Gitlab app poc

.github/workflows/lint.yml

@@ -0,0 +1,21 @@
+name: Lint
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+      - name: Lint
+        run: | 
+            make install/all
+            make lint/all

.github/workflows/release-framework.yml

@@ -0,0 +1,31 @@
+name: Release framework
+on:
+  push:
+    tags:
+      - v*
+  workflow_dispatch:
+
+env:
+  PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+  PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }}
+jobs:
+  publish_ocean:
+    name: Publish Ocean
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+      - name: Make install and build
+        run: | 
+            make install
+            make build
+      - name: Publish package to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          user: ${{ env.PYPI_USERNAME }}
+          password: ${{ env.PYPI_PASSWORD }}
+          packages-dir: ${{github.workspace}}/dist

.github/workflows/release-integrations.yml

@@ -0,0 +1,67 @@
+name: Release integrations
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+jobs:
+  release-all:
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ secrets.DOCKER_MACHINE_USER }}
+          password: ${{ secrets.DOCKER_MACHINE_TOKEN }}
+      - name: Build and push Docker image
+        run: |
+          GHCR_TOKEN=$(echo ${{ secrets.GITHUB_TOKEN }} | base64)
+
+          files=$(find integrations/*/.port -name "spec.yaml")
+          for file in $files; do
+            # Get the type from ocean-spec.yaml
+            type=$(yq eval '.type' "$file")
+            folder=$(dirname "$file")
+            # Get the version from ocean-spec.yaml
+            version=$(yq eval '.version' "$file")
+            # Check if the version exists in the ghcr.io registry
+            MANIFEST=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: Bearer ${GHCR_TOKEN}" "https://ghcr.io/v2/port-labs/port-ocean-$type/manifests/$version")
+            if [ "$MANIFEST" = "200" ]; then            
+              echo "Image already exists in $repository: port-ocean-$type:$version"
+            else
+              echo "Building and pushing new image: port-ocean-$type:$version"
+              docker build -t "ghcr.io/port-labs/port-ocean-$type:$version" -t "ghcr.io/port-labs/port-ocean-$type:latest" "$folder/.."
+              docker push "ghcr.io/port-labs/port-ocean-$type" --all-tags
+            fi
+          done
+      - name: Configure AWS Credentials ?
+        id: aws-credentials
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+      - name: Upload specifications to s3
+        run: |
+          # Temporary file to store the concatenated YAML content
+          temp_file="temp.yaml"
+          # Output file name
+          output_file="index.json"
+          # AWS S3 bucket details
+          aws_s3_bucket="ocean-registry"
+          # Find all ocean-spec.yaml files under the specified directory
+          find integrations/*/.port -type f -name "spec.yaml" > file_list.txt
+          # Concatenate the YAML files into a temporary file
+          while IFS= read -r file; do
+              echo "- " >> "$temp_file"
+              sed 's/^/  /' "$file" >> "$temp_file"
+          done < file_list.txt
+          yq -j . < "$temp_file" > "$output_file"
+          aws s3 cp "$output_file" "s3://$aws_s3_bucket/$output_file"

.gitignore

@@ -0,0 +1,160 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# Pycharm
+.idea
+!.idea/watcherTasks.xml
+
+# VSCode
+.vscode/

CONTRIBUTING.md

@@ -0,0 +1,31 @@
+# Contributing to Ocean
+
+Thank you for considering contributing to our data integration framework! Your contributions play a crucial role in improving the functionality, scalability, and usability of the framework. This readme aims to explain the difference between contributing to the framework itself and contributing a new integration.
+
+## Contributing to the Framework
+
+Contributing to the framework involves actively participating in its development and improvement. By contributing, you can help enhance its core features, fix bugs, optimize performance, and ensure compatibility with various data sources and formats. Here are some ways you can contribute to the framework:
+
+1. **Feature Development**: You can propose and implement new features that align with the framework's objectives. Whether it's introducing advanced data transformation capabilities, supporting new data storage systems, or improving the user interface, your ideas and contributions are valuable.
+
+2. **Bug Fixes**: If you encounter any bugs or issues while using the framework, you can contribute by investigating the problem, identifying the root cause, and submitting fixes. By addressing these issues, you ensure a more stable and reliable framework for all users.
+
+3. **Performance Optimization**: Improving the framework's performance is always a priority. You can contribute by optimizing algorithms, reducing resource usage, and implementing caching mechanisms, resulting in faster and more efficient data integration processes.
+CON
+4. **Documentation**: Clear and comprehensive documentation is crucial for users to understand and leverage the framework's capabilities. You can help by improving existing documentation, writing new guides, and providing code examples that demonstrate various integration scenarios.
+
+5. **Code Reviews**: Reviewing code submissions from other contributors is an essential part of maintaining code quality. By participating in code reviews, you can provide feedback, suggest improvements, and ensure adherence to coding standards.
+
+## Contributing a New Integration
+
+Apart from contributing to the framework itself, you can also contribute by adding new integrations to expand its compatibility and usefulness. Integrations allow the framework to connect with different data sources, APIs, and services. Here's how you can contribute a new integration:
+
+1. **Integration Proposal**: Start by proposing the integration you would like to add. Provide a clear description of the data source or service, explain how it aligns with the framework's goals, and outline the benefits it would bring to users.
+
+2. **Integration Implementation**: Once your proposal is accepted, you can begin implementing the integration. This involves writing code to establish connections, retrieve data, and handle any necessary authentication or authorization mechanisms.
+
+3. **Testing and Documentation**: Thoroughly test the integration to ensure its reliability and functionality. Additionally, document the integration by providing usage instructions, configuration details, and any specific considerations users should be aware of.
+
+4. **Integration Submission**: Finally, submit your integration code and documentation to the framework's repository for review. Collaborate with other contributors to address feedback and make any necessary improvements.
+
+By contributing a new integration, you enable users of the framework to seamlessly integrate with additional data sources and services, expanding its versatility and usefulness.