fix: delete pipeline config on pipeline delete

dataworkspace/dataworkspace/apps/datasets/models.py

@@ -2975,6 +2975,10 @@ def save(self, force_insert=False, force_update=False, using=None, update_fields
         super().save(force_insert, force_update, using, update_fields)
         self.save_dataflow_pipeline_config()
 
+    def delete(self, using=None, keep_parents=False):
+        self.delete_dataflow_pipeline_config()
+        super().delete(using, keep_parents)
+
     def get_config_file_path(self):
         return f"{settings.DATAFLOW_PIPELINES_PREFIX}/pipeline-{self.id}.json"
 
@@ -2987,6 +2991,7 @@ def save_dataflow_pipeline_config(self):
             Body=json.dumps(
                 {
                     "id": self.id,
+                    "dag_id": self.dag_id,
                     "type": self.type,
                     "schedule": self.schedule,
                     "schema": schema_name,
@@ -2997,6 +3002,15 @@ def save_dataflow_pipeline_config(self):
             ),
         )
 
+    def delete_dataflow_pipeline_config(self):
+        client = get_s3_client()
+        try:
+            client.delete_object(
+                Bucket=settings.AWS_UPLOADS_BUCKET, Key=self.get_config_file_path()
+            )
+        except ClientError:
+            pass
+
 
 class PipelineVersion(TimeStampedModel):
     pipeline = models.ForeignKey(Pipeline, on_delete=models.CASCADE)