From b081ef3d86163b5634be83a4d5d5122ec0529e81 Mon Sep 17 00:00:00 2001 From: guillaumebarat Date: Mon, 31 Mar 2025 11:17:24 +1000 Subject: [PATCH 1/2] New step to remove a file inside the sftp server --- .../local/step/connector_sftp_delete_file.php | 129 ++++++++++++++++++ classes/local/step/sftp_trait.php | 77 +++++++++-- lang/en/tool_dataflows.php | 2 + lib.php | 1 + version.php | 2 +- 5 files changed, 200 insertions(+), 11 deletions(-) create mode 100644 classes/local/step/connector_sftp_delete_file.php diff --git a/classes/local/step/connector_sftp_delete_file.php b/classes/local/step/connector_sftp_delete_file.php new file mode 100644 index 00000000..5a7e7983 --- /dev/null +++ b/classes/local/step/connector_sftp_delete_file.php @@ -0,0 +1,129 @@ +. + +namespace tool_dataflows\local\step; + +use tool_dataflows\helper; + +/** + * SFTP connector step type. + * + * Uses phpseclib. See https://phpseclib.com + * + * @package tool_dataflows + * @author Guillaume Barat + * @copyright 2025, Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class connector_sftp_delete_file extends connector_sftp { + use sftp_trait{ + sftp_trait::form_define_fields as sftp_form_define_fields; + sftp_trait::form_add_custom_inputs as sftp_form_add_custom_inputs; + sftp_trait::validate_for_run as sftp_validate_for_run; + sftp_trait::validate_config as sftp_validate_config; + + + } + + /** + * Return the definition of the fields available in this form. + * + * @param string $behaviour 'delete' or something else. + * @return array + */ + public static function form_define_fields($behaviour = 'delete'): array { + $fields = self::sftp_form_define_fields('delete'); + return $fields; + } + + /** + * Custom elements for editing the step. + * + * @param \MoodleQuickForm $mform + * @param string $behaviour default to the step. + */ + public function form_add_custom_inputs(\MoodleQuickForm &$mform, $behaviour = 'delete') { + $this->sftp_form_add_custom_inputs($mform, 'delete'); + } + + /** + * Executes the step + * + * Performs an SFTP call according to config parameters. + * + * @param mixed|null $input + * @return mixed + */ + public function execute($input = null) { + $stepvars = $this->get_variables(); + $config = $stepvars->get('config'); + + // At this point we need to disconnect once we are finished. + try { + // Skip if it is a dry run. + if ($this->is_dry_run() && $this->has_side_effect()) { + return $input; + } + + $sftp = $this->init_sftp($config); + + $sourceisremote = helper::path_is_scheme($config->source, self::$sftpprefix); + $sourcepath = $this->resolve_path($config->source); + + // Deleting from remote. + if ($sourceisremote) { + $this->delete_from_remote($sftp, $sourcepath); + return $input; + } + } catch (\Throwable $e) { + $this->enginestep->log->error($e->getMessage()); + if (isset($sftp)) { + $sftp->disconnect(); + } + throw new \moodle_exception($e->getMessage(), 'tool_dataflows'); + } + + return $input; + } + + + /** + * Perform any extra validation that is required only for runs. + * + * @param string $behaviour + * @return true|array Will return true or an array of errors. + */ + public function validate_for_run($behaviour = 'copy') { + $sftperrors = $this->sftp_validate_for_run('delete'); + if ($sftperrors === true) { + $sftperrors = []; + } + return $sftperrors ?: true; + } + + /** + * Validate the configuration settings. + * + * @param object $config + * @param string $behaviour + * @return true|\lang_string[] true if valid, an array of errors otherwise + */ + public function validate_config($config, $behaviour = 'delete') { + $errors = $this->sftp_validate_config($config, 'delete'); + return $errors ?: true; + } + +} diff --git a/classes/local/step/sftp_trait.php b/classes/local/step/sftp_trait.php index 35d32e8d..affad1ae 100644 --- a/classes/local/step/sftp_trait.php +++ b/classes/local/step/sftp_trait.php @@ -80,6 +80,11 @@ public static function form_define_fields($behaviour = 'copy'): array { 'target' => ['type' => PARAM_TEXT, 'required' => true], ]); } + if ($behaviour === 'delete') { + $fields = array_merge($fields, [ + 'source' => ['type' => PARAM_TEXT, 'required' => true], + ]); + } return $fields; } @@ -131,6 +136,16 @@ public function form_add_custom_inputs(\MoodleQuickForm &$mform, $behaviour = 'c ) ); } + if ($behaviour === 'delete') { + $mform->addElement('text', 'config_source', get_string('connector_sftp:source', 'tool_dataflows')); + $mform->addElement('static', 'config_source_desc', '', get_string('connector_sftp:source_desc', 'tool_dataflows'). + \html_writer::nonempty_tag( + 'pre', + get_string('connector_sftp:path_example', 'tool_dataflows'). + get_string('path_help_examples', 'tool_dataflows') + ) + ); + } } /** @@ -190,19 +205,42 @@ public function validate_config($config, $behaviour = 'copy') { ); } } + // Delete step checks. + if ($behaviour === 'delete') { + if (empty($config->source)) { + $errors['config_source'] = get_string( + 'config_field_missing', + 'tool_dataflows', + get_string('connector_sftp:source', 'tool_dataflows'), + true + ); + } + } $hasremote = true; // Check that at least one file config has an sftp:// scheme. - if (!empty($config->source) && !empty($config->target)) { - // Check if the source or target is an expression, and evaluate it if required. - $sourceremote = helper::path_has_scheme($config->source, self::$sftpprefix); - $targetremote = helper::path_has_scheme($config->target, self::$sftpprefix); - $hasremote = $sourceremote || $targetremote; - } - if (!$hasremote) { - $errormsg = get_string('connector_sftp:missing_remote', 'tool_dataflows', null, true); - $errors['config_source'] = $errors['config_source'] ?? $errormsg; - $errors['config_target'] = $errors['config_target'] ?? $errormsg; + if ($behaviour === 'delete') { + if (!empty($config->source)) { + // Check if the source or target is an expression, and evaluate it if required. + $sourceremote = helper::path_has_scheme($config->source, self::$sftpprefix); + $hasremote = $sourceremote; + } + if (!$hasremote) { + $errormsg = get_string('connector_sftp:missing_remote', 'tool_dataflows', null, true); + $errors['config_source'] = $errors['config_source'] ?? $errormsg; + } + } else { + if (!empty($config->source) && !empty($config->target)) { + // Check if the source or target is an expression, and evaluate it if required. + $sourceremote = helper::path_has_scheme($config->source, self::$sftpprefix); + $targetremote = helper::path_has_scheme($config->target, self::$sftpprefix); + $hasremote = $sourceremote || $targetremote; + } + if (!$hasremote) { + $errormsg = get_string('connector_sftp:missing_remote', 'tool_dataflows', null, true); + $errors['config_source'] = $errors['config_source'] ?? $errormsg; + $errors['config_target'] = $errors['config_target'] ?? $errormsg; + } } return empty($errors) ? true : $errors; @@ -230,6 +268,12 @@ public function validate_for_run($behaviour = 'copy') { $errors['config_target'] = $error; } } + if ($behaviour === 'delete') { + $error = helper::path_validate($config->source); + if ($error !== true) { + $errors['config_source'] = $error; + } + } if (!empty($config->privkeyfile)) { $error = helper::path_validate($config->privkeyfile); if ($error !== true) { @@ -399,6 +443,19 @@ private function copy_remote_to_remote(SFTP $sftp, string $sourcepath, string $t $this->upload($sftp, $tmppath, $targetpath); } + /** + * Delete a file from one remote source + * + * @param SFTP $sftp + * @param string $sourcepath + */ + private function delete_from_remote(SFTP $sftp, string $sourcepath) { + $this->log("Deleting from '$sourcepath'"); + if (!$sftp->delete($sourcepath, false)) { + throw new \moodle_exception('connector_sftp:delete_fail', 'tool_dataflows', '', $sftp->getLastSFTPError()); + } + } + /** * Lists files in a directory * diff --git a/lang/en/tool_dataflows.php b/lang/en/tool_dataflows.php index bca85672..d9a906e6 100644 --- a/lang/en/tool_dataflows.php +++ b/lang/en/tool_dataflows.php @@ -158,6 +158,7 @@ $string['step_name_connector_set_variable'] = 'Set variable'; $string['step_name_connector_set_multiple_variables'] = 'Set multiple variables'; $string['step_name_connector_sftp'] = 'SFTP file copy'; +$string['step_name_connector_sftp_delete_file'] = 'SFTP file to delete'; $string['step_name_connector_sns_notify'] = 'AWS-SNS Notification'; $string['step_name_connector_wait'] = 'Wait'; $string['step_name_flow_abort'] = 'Abort'; @@ -485,6 +486,7 @@ $string['connector_sftp:hostpubkey'] = 'Host public key'; $string['connector_sftp:hostpubkey_desc'] = 'Public key that must match the one returned by the host. If empty, it will be set on the first connection.'; $string['connector_sftp:copy_fail'] = 'Copy failed: \'{$a}\'.'; +$string['connector_sftp:delete_fail'] = 'Delete failed: \'{$a}\'.'; $string['connector_sftp:missing_remote'] = 'At least one of source/target must be remote.'; $string['connector_sftp:pubkeyfile'] = 'Public key file'; $string['connector_sftp:privkeyfile'] = 'Private key file'; diff --git a/lib.php b/lib.php index 0198f912..c5c20101 100644 --- a/lib.php +++ b/lib.php @@ -60,6 +60,7 @@ function tool_dataflows_step_types() { new step\connector_s3, new step\connector_set_variable, new step\connector_sftp, + new step\connector_sftp_delete_file, new step\connector_sql, new step\connector_update_user, new step\connector_sftp_directory_file_list, diff --git a/version.php b/version.php index 6fcc1130..c6b5fba3 100644 --- a/version.php +++ b/version.php @@ -25,7 +25,7 @@ defined('MOODLE_INTERNAL') || die(); -$plugin->version = 2025021100; +$plugin->version = 2025033100; $plugin->release = 2025021100; $plugin->requires = 2024100700; // Our lowest supported Moodle (4.5.0). $plugin->supported = [405, 405]; From b4e64cace9d23039c117fe84199786b831ba26eb Mon Sep 17 00:00:00 2001 From: guillaumebarat Date: Mon, 31 Mar 2025 17:04:13 +1000 Subject: [PATCH 2/2] Update readme.md: change order of version in table branches --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a4c8e3b1..55776ad3 100644 --- a/README.md +++ b/README.md @@ -30,8 +30,8 @@ Dataflows is a generic workflow and processing engine which can be configured to | Moodle version | Branch | PHP | |----------------|-------------------|-----------| -| Moodle 4.1-4.2 | MOODLE_401_STABLE | 7.4 | | Moodle 4.5 | MOODLE_405_STABLE | 8.1 - 8.3 | +| Moodle 4.1-4.2 | MOODLE_401_STABLE | 7.4 | | Totara 10+ | MOODLE_35_STABLE | 7.1 - 7.4 | Note: Moodle 402 is supported with PHP 8.0 maximum at the moment