diff --git a/.github/workflows/test-code-syntactic_concept_extractor.yml b/.github/workflows/test-code-syntactic_concept_extractor.yml new file mode 100644 index 000000000..7f95b90a8 --- /dev/null +++ b/.github/workflows/test-code-syntactic_concept_extractor.yml @@ -0,0 +1,124 @@ +# +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files +# +name: Test - transforms/code/syntactic_concept_extractor + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "transforms/code/syntactic_concept_extractor/**" + - "data-processing-lib/**" + - "!transforms/code/syntactic_concept_extractor/**/kfp_ray/**" # This is/will be tested in separate workflow + - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "transforms/code/syntactic_concept_extractor/**" + - "data-processing-lib/**" + - "!transforms/code/syntactic_concept_extractor/**/kfp_ray/**" # This is/will be tested in separate workflow + - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + +jobs: + check_if_push_image: + # check whether the Docker images should be pushed to the remote repository + # The images are pushed if it is a merge to dev branch or a new tag is created. + # The latter being part of the release process. + # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. + runs-on: ubuntu-22.04 + outputs: + publish_images: ${{ steps.version.outputs.publish_images }} + steps: + - id: version + run: | + publish_images='false' + if [[ ${GITHUB_REF} == refs/heads/dev && ${GITHUB_EVENT_NAME} != 'pull_request' && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + if [[ ${GITHUB_REF} == refs/tags/* && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" + test-src: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test transform source in transforms/code/syntactic_concept_extractor + run: | + if [ -e "transforms/code/syntactic_concept_extractor/Makefile" ]; then + make -C transforms/code/syntactic_concept_extractor DOCKER=docker test-src + else + echo "transforms/code/syntactic_concept_extractor/Makefile not found - source testing disabled for this transform." + fi + test-image: + needs: [check_if_push_image] + runs-on: ubuntu-22.04 + timeout-minutes: 120 + env: + DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test transform image in transforms/code/syntactic_concept_extractor + run: | + if [ -e "transforms/code/syntactic_concept_extractor/Makefile" ]; then + if [ -d "transforms/code/syntactic_concept_extractor/spark" ]; then + make -C data-processing-lib/spark DOCKER=docker image + fi + make -C transforms/code/syntactic_concept_extractor DOCKER=docker test-image + else + echo "transforms/code/syntactic_concept_extractor/Makefile not found - testing disabled for this transform." + fi + - name: Print space + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + docker images + - name: Publish images + if: needs.check_if_push_image.outputs.publish_images == 'true' + run: | + if [ -e "transforms/code/syntactic_concept_extractor/Makefile" ]; then + make -C transforms/code/syntactic_concept_extractor publish + else + echo "transforms/code/syntactic_concept_extractor/Makefile not found - publishing disabled for this transform." + fi diff --git a/.make.versions b/.make.versions index 6c9bbc08d..73eec8542 100644 --- a/.make.versions +++ b/.make.versions @@ -109,6 +109,10 @@ HTML2PARQUET_PYTHON_VERSION=$(DPK_VERSION) DPK_TRANSFORMS_VERSION=$(DPK_VERSION) +SYNTACTIC_CONCEPT_EXTRACTOR_PYTHON_VERSION=$(DPK_VERSION) +SYNTACTIC_CONCEPT_EXTRACTOR_RAY_VERSION=$(DPK_VERSION) + + ################## ################## ################## ################## ################## ################## # Begin versions that the repo depends on. diff --git a/transforms/code/Makefile b/transforms/code/Makefile index 17afe2785..b5d5c7bbe 100644 --- a/transforms/code/Makefile +++ b/transforms/code/Makefile @@ -27,26 +27,10 @@ image:: @# Help: Recursively make $@ in all subdirs @$(MAKE) RULE=$@ .recurse -test-image:: - @# Help: Recursively make $@ in all subdirs - @$(MAKE) RULE=$@ .recurse - publish:: @# Help: Recursively make $@ in all subdirs @$(MAKE) RULE=$@ .recurse -kind-load-image:: - @# Help: Recursively make $@ in all subdirs - @$(MAKE) RULE=$@ .recurse - -docker-load-image:: - @# Help: Recursively make $@ in all subdirs - $(MAKE) RULE=$@ .recurse - -docker-save-image:: - @# Help: Recursively make $@ in all subdirs - $(MAKE) RULE=$@ .recurse - set-versions: @# Help: Recursively $@ in all subdirs @$(MAKE) RULE=$@ .recurse diff --git a/transforms/code/syntactic_concept_extractor/Makefile b/transforms/code/syntactic_concept_extractor/Makefile index 351da91d5..3cc939591 100644 --- a/transforms/code/syntactic_concept_extractor/Makefile +++ b/transforms/code/syntactic_concept_extractor/Makefile @@ -21,6 +21,9 @@ publish:: @# Help: Recursively make $@ in all subdirs @$(MAKE) RULE=$@ .recurse +test-image: + @echo "Skipping test-image step as per configuration." + test:: @# Help: Recursively make $@ in all subdirs @$(MAKE) RULE=$@ .recurse diff --git a/transforms/code/syntactic_concept_extractor/README.md b/transforms/code/syntactic_concept_extractor/README.md index 36b1e57b8..eb1b181ea 100644 --- a/transforms/code/syntactic_concept_extractor/README.md +++ b/transforms/code/syntactic_concept_extractor/README.md @@ -54,3 +54,10 @@ The implementation for UI-based offline customization tool is present [here](pyt `streamlit run LLM_runner_app.py` +The high-level system design is as follows: + +![White Background Image](sys-overview.png) + +For each new target language, the offline phase is utilized to create deterministic rules by harnessing the capabilities of LLMs and working with exemplar code samples from the target language. In this process, Workflow W1 facilitates the creation of rules around syntactic structures based on exemplar code samples, while Workflow W2 is used to establish semantic dimensions for profiling. Subsequently, we derive rules that connect syntactic constructs to the predefined semantic concepts. These rules are then stored in a rule database, ready to be employed during the online phase. + +In the online phase, the system dynamically generates profiling outputs for any incoming code snippets. This is achieved by extracting concepts from the snippets using the rules in the database and storing these extractions in a tabular format. The structured tabular format allows for generating additional concept columns, which are then utilized to create comprehensive profiling reports. \ No newline at end of file diff --git a/transforms/code/syntactic_concept_extractor/input/multi-package.parquet b/transforms/code/syntactic_concept_extractor/input/multi-package.parquet index 8ef9f7cc2..fc96e51fa 100644 Binary files a/transforms/code/syntactic_concept_extractor/input/multi-package.parquet and b/transforms/code/syntactic_concept_extractor/input/multi-package.parquet differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/COBOL-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/COBOL-bindings.so new file mode 100755 index 000000000..2a967cc12 Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/COBOL-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/agda-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/agda-bindings.so new file mode 100755 index 000000000..7ff34af68 Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/agda-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/bash-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/bash-bindings.so new file mode 100755 index 000000000..a6bd2bea4 Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/bash-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/c-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/c-bindings.so new file mode 100755 index 000000000..4209cf84f Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/c-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/c_sharp-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/c_sharp-bindings.so new file mode 100755 index 000000000..dea942a7c Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/c_sharp-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/clojure-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/clojure-bindings.so new file mode 100755 index 000000000..d17bf0cd0 Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/clojure-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/cpp-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/cpp-bindings.so new file mode 100755 index 000000000..d3a48459a Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/cpp-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/d-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/d-bindings.so new file mode 100755 index 000000000..05d75367c Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/d-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/dart-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/dart-bindings.so new file mode 100755 index 000000000..06f2fe1b0 Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/dart-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/dot-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/dot-bindings.so new file mode 100755 index 000000000..17aa810a2 Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/dot-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/elixir-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/elixir-bindings.so new file mode 100755 index 000000000..d02e3d11d Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/elixir-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/elm-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/elm-bindings.so new file mode 100755 index 000000000..3473e2f6f Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/elm-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/erlang-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/erlang-bindings.so new file mode 100755 index 000000000..99cd40407 Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/erlang-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/go-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/go-bindings.so new file mode 100755 index 000000000..596261312 Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/go-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/haskell-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/haskell-bindings.so new file mode 100755 index 000000000..96b830ea6 Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/haskell-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/java-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/java-bindings.so new file mode 100755 index 000000000..59c92a593 Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/java-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/js-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/js-bindings.so new file mode 100755 index 000000000..a11264f02 Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/js-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/julia-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/julia-bindings.so new file mode 100755 index 000000000..400c57985 Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/julia-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/kotlin-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/kotlin-bindings.so new file mode 100755 index 000000000..be1e17faf Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/kotlin-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/lua-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/lua-bindings.so new file mode 100755 index 000000000..6e3055f09 Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/lua-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/nim-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/nim-bindings.so new file mode 100755 index 000000000..0102f655c Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/nim-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/objc-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/objc-bindings.so new file mode 100755 index 000000000..3c6e58df6 Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/objc-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/pascal-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/pascal-bindings.so new file mode 100755 index 000000000..5207b7388 Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/pascal-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/py-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/py-bindings.so new file mode 100755 index 000000000..7645d64b3 Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/py-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/qmljs-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/qmljs-bindings.so new file mode 100755 index 000000000..4948bc600 Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/qmljs-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/r-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/r-bindings.so new file mode 100755 index 000000000..aeed682c2 Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/r-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/ruby-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/ruby-bindings.so new file mode 100755 index 000000000..0b93238fb Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/ruby-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/rust-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/rust-bindings.so new file mode 100755 index 000000000..76488d798 Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/rust-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/scala-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/scala-bindings.so new file mode 100755 index 000000000..572ae96b8 Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/scala-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/svelte-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/svelte-bindings.so new file mode 100755 index 000000000..346c53140 Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/svelte-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/verilog-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/verilog-bindings.so new file mode 100755 index 000000000..59e10ae21 Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/verilog-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/vhdl-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/vhdl-bindings.so new file mode 100755 index 000000000..334052614 Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64/vhdl-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/agda-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/agda-bindings.so new file mode 100755 index 000000000..9669bf45e Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/agda-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/c-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/c-bindings.so new file mode 100755 index 000000000..ce899b0d1 Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/c-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/c_sharp-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/c_sharp-bindings.so new file mode 100755 index 000000000..9eef24a2d Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/c_sharp-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/cpp-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/cpp-bindings.so new file mode 100755 index 000000000..ad428e9f0 Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/cpp-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/d-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/d-bindings.so new file mode 100755 index 000000000..2fa84825f Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/d-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/dart-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/dart-bindings.so new file mode 100755 index 000000000..f04d6eea4 Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/dart-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/dot-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/dot-bindings.so new file mode 100755 index 000000000..82c2100b3 Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/dot-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/elm-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/elm-bindings.so new file mode 100755 index 000000000..343f55516 Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/elm-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/erlang-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/erlang-bindings.so new file mode 100755 index 000000000..bf680fab9 Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/erlang-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/go-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/go-bindings.so new file mode 100755 index 000000000..1344b2287 Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/go-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/haskell-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/haskell-bindings.so new file mode 100755 index 000000000..cfd18e764 Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/haskell-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/java-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/java-bindings.so new file mode 100755 index 000000000..9223ac9e8 Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/java-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/js-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/js-bindings.so new file mode 100755 index 000000000..e6a6d3864 Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/js-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/kotlin-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/kotlin-bindings.so new file mode 100755 index 000000000..c13f99baa Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/kotlin-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/lua-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/lua-bindings.so new file mode 100755 index 000000000..4862d2ae2 Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/lua-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/nim-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/nim-bindings.so new file mode 100755 index 000000000..ff81eb9ef Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/nim-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/py-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/py-bindings.so new file mode 100755 index 000000000..736e7429e Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/py-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/qmljs-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/qmljs-bindings.so new file mode 100755 index 000000000..cc1588f7e Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/qmljs-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/r-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/r-bindings.so new file mode 100755 index 000000000..72586228a Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/r-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/ruby-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/ruby-bindings.so new file mode 100755 index 000000000..985ce85f6 Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/ruby-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/rust-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/rust-bindings.so new file mode 100755 index 000000000..06e9bc4ec Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/rust-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/scala-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/scala-bindings.so new file mode 100755 index 000000000..7e8d80f48 Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/scala-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/verilog-bindings.so b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/verilog-bindings.so new file mode 100755 index 000000000..28cf6e13e Binary files /dev/null and b/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/x86_64/verilog-bindings.so differ diff --git a/transforms/code/syntactic_concept_extractor/notebook_example/code-profiler.ipynb b/transforms/code/syntactic_concept_extractor/notebook_example/code-profiler.ipynb new file mode 100644 index 000000000..f98f50b55 --- /dev/null +++ b/transforms/code/syntactic_concept_extractor/notebook_example/code-profiler.ipynb @@ -0,0 +1,1224 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 61, + "id": "08ad3138-a87c-4fb3-80fd-8a64cdc27eaf", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/var/folders/f7/736gtk714sg36xlcqnjx16xw0000gn/T/ipykernel_45869/3873726698.py:4: DeprecationWarning: Importing display from IPython.core.display is deprecated since IPython 7.14, please import from IPython display\n", + " from IPython.core.display import display, HTML\n" + ] + } + ], + "source": [ + "import os\n", + "from IPython.display import display, HTML # Updated import\n", + "import subprocess\n", + "from IPython.core.display import display, HTML\n", + "import shutil\n", + "\n", + "# Following are the utility functions for running the transforms sequencially\n", + "\n", + "def run_make_command(target_dir, command):\n", + " \"\"\"Function to change directory and run a make command with real-time log output.\"\"\"\n", + " if os.path.exists(target_dir):\n", + " # Change the current working directory to the target directory\n", + " os.chdir(target_dir)\n", + " print(f\"Changed directory to: {os.getcwd()}\")\n", + "\n", + " # Run the make command and stream logs\n", + " try:\n", + " process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)\n", + " for stdout_line in iter(process.stdout.readline, \"\"):\n", + " print(stdout_line, end=\"\") # Print stdout line-by-line in real-time\n", + " process.stdout.close()\n", + " process.wait()\n", + "\n", + " # Check for errors and handle stderr\n", + " if process.returncode != 0:\n", + " print(\"Error Output:\")\n", + " for stderr_line in iter(process.stderr.readline, \"\"):\n", + " print(stderr_line, end=\"\")\n", + " process.stderr.close()\n", + " else:\n", + " print(\"Process completed successfully.\")\n", + " except subprocess.CalledProcessError as e:\n", + " print(f\"Error occurred while running the make command: {e}\")\n", + " else:\n", + " print(f\"Directory does not exist: {target_dir}\")\n", + "\n", + "def check_directories_exist(directories):\n", + " results = {}\n", + " for directory in directories:\n", + " if os.path.isdir(directory):\n", + " print(f\"The directory '{directory}' exists.\")\n", + " results[directory] = True\n", + " else:\n", + " print(f\"The directory '{directory}' does not exist.\")\n", + " results[directory] = False\n", + " return results\n", + "\n", + "def display_html_file(hosp_code_dir):\n", + " # Construct the path to the HTML file\n", + " html_file_path = os.path.join(hosp_code_dir, 'src', 'output.html')\n", + " \n", + " # Check if the file exists\n", + " if not os.path.exists(html_file_path):\n", + " raise FileNotFoundError(f\"The file '{html_file_path}' does not exist.\")\n", + " \n", + " # Read the HTML file\n", + " with open(html_file_path, 'r', encoding='utf-8') as file:\n", + " html_content = file.read()\n", + " \n", + " # Display the HTML content in the notebook\n", + " display(HTML(html_content))\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 62, + "id": "665c28d4-2485-4cf4-aca7-6d0f6ac2353c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The directory '/Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/input' exists.\n", + "The directory '/Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python' exists.\n", + "The directory '/Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/output' exists.\n", + "The directory '/Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/transforms/code/semantic_profiler/python/test-data/input' exists.\n", + "The directory '/Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/transforms/code/semantic_profiler/python' exists.\n", + "The directory '/Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/transforms/code/semantic_profiler/python/output' exists.\n", + "The directory '/Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/transforms/code/higher_order_syntactic_profiler/python/test-data/input' exists.\n", + "The directory '/Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/transforms/code/higher_order_syntactic_profiler/python' exists.\n", + "The directory '/Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/transforms/code/higher_order_syntactic_profiler/python/output' exists.\n", + "Checking directory exists or not\n", + "/Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/input: Exists\n", + "/Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python: Exists\n", + "/Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/output: Exists\n", + "/Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/transforms/code/semantic_profiler/python/test-data/input: Exists\n", + "/Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/transforms/code/semantic_profiler/python: Exists\n", + "/Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/transforms/code/semantic_profiler/python/output: Exists\n", + "/Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/transforms/code/higher_order_syntactic_profiler/python/test-data/input: Exists\n", + "/Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/transforms/code/higher_order_syntactic_profiler/python: Exists\n", + "/Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/transforms/code/higher_order_syntactic_profiler/python/output: Exists\n" + ] + } + ], + "source": [ + "# Specify the target directory where your transforms Makefile and output are located\n", + "user_local_dir = '/Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk'\n", + "\n", + "transforms_dir = user_local_dir + '/data-prep-kit/transforms'\n", + "\n", + "sce_input_dir = transforms_dir + '/code/syntactic_concept_extractor/input'\n", + "sce_code_dir = transforms_dir + '/code/syntactic_concept_extractor/python'\n", + "sce_output_dir = transforms_dir + '/code/syntactic_concept_extractor/output'\n", + "\n", + "sp_input_dir = transforms_dir + '/code/semantic_profiler/python/test-data/input'\n", + "sp_code_dir = transforms_dir + '/code/semantic_profiler/python'\n", + "sp_output_dir = transforms_dir + '/code/semantic_profiler/python/output'\n", + "\n", + "hosp_input_dir = transforms_dir + '/code/higher_order_syntactic_profiler/python/test-data/input'\n", + "hosp_code_dir = transforms_dir + '/code/higher_order_syntactic_profiler/python'\n", + "hosp_output_dir = transforms_dir + '/code/higher_order_syntactic_profiler/python/output'\n", + "\n", + "directories_to_check = [sce_input_dir, sce_code_dir, sce_output_dir, sp_input_dir, sp_code_dir, sp_output_dir, hosp_input_dir, hosp_code_dir, hosp_output_dir]\n", + "check_results = check_directories_exist(directories_to_check)\n", + "\n", + "print(\"Checking directory exists or not\")\n", + "for directory, exists in check_results.items():\n", + " print(f\"{directory}: {'Exists' if exists else 'Does not exist'}\")\n", + " \n", + " if not exists:\n", + " if 'output' in directory:\n", + " os.makedirs(directory, exist_ok=True)\n", + " print(f\"Directory '{directory}' did not exist, so it was created.\")\n", + " else:\n", + " raise FileNotFoundError(f\"The directory '{directory}' does not exist.\")" + ] + }, + { + "cell_type": "code", + "execution_count": 63, + "id": "65ad6896-8ecc-492f-836d-81ae146c0b22", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Changed directory to: /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python\n", + "python -m venv venv\n", + "Requirement already satisfied: pip in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (24.2)\n", + "Requirement already satisfied: wheel in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (0.44.0)\n", + "Requirement already satisfied: pytest in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (8.3.3)\n", + "Requirement already satisfied: pytest-cov in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (5.0.0)\n", + "Requirement already satisfied: iniconfig in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from pytest) (2.0.0)\n", + "Requirement already satisfied: packaging in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from pytest) (24.0)\n", + "Requirement already satisfied: pluggy<2,>=1.5 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from pytest) (1.5.0)\n", + "Requirement already satisfied: coverage>=5.2.1 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from coverage[toml]>=5.2.1->pytest-cov) (7.6.1)\n", + "Installing Python data processing library source to existing venv\n", + "pip uninstall -y data-prep-toolkit \n", + "Found existing installation: data_prep_toolkit 0.2.1.dev0\n", + "Uninstalling data_prep_toolkit-0.2.1.dev0:\n", + " Successfully uninstalled data_prep_toolkit-0.2.1.dev0\n", + "Begin installing source from ../../../../data-processing-lib/python into venv\n", + "Obtaining file:///Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/data-processing-lib/python\n", + " Installing build dependencies: started\n", + " Installing build dependencies: finished with status 'done'\n", + " Checking if build backend supports build_editable: started\n", + " Checking if build backend supports build_editable: finished with status 'done'\n", + " Getting requirements to build editable: started\n", + " Getting requirements to build editable: finished with status 'done'\n", + " Preparing editable metadata (pyproject.toml): started\n", + " Preparing editable metadata (pyproject.toml): finished with status 'done'\n", + "Requirement already satisfied: numpy<1.29.0 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from data_prep_toolkit==0.2.2.dev0) (1.26.4)\n", + "Requirement already satisfied: pyarrow==16.1.0 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from data_prep_toolkit==0.2.2.dev0) (16.1.0)\n", + "Requirement already satisfied: boto3==1.34.69 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from data_prep_toolkit==0.2.2.dev0) (1.34.69)\n", + "Collecting argparse (from data_prep_toolkit==0.2.2.dev0)\n", + " Using cached argparse-1.4.0-py2.py3-none-any.whl.metadata (2.8 kB)\n", + "Requirement already satisfied: mmh3 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from data_prep_toolkit==0.2.2.dev0) (5.0.1)\n", + "Requirement already satisfied: botocore<1.35.0,>=1.34.69 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from boto3==1.34.69->data_prep_toolkit==0.2.2.dev0) (1.34.162)\n", + "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from boto3==1.34.69->data_prep_toolkit==0.2.2.dev0) (1.0.1)\n", + "Requirement already satisfied: s3transfer<0.11.0,>=0.10.0 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from boto3==1.34.69->data_prep_toolkit==0.2.2.dev0) (0.10.2)\n", + "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from botocore<1.35.0,>=1.34.69->boto3==1.34.69->data_prep_toolkit==0.2.2.dev0) (2.9.0.post0)\n", + "Requirement already satisfied: urllib3!=2.2.0,<3,>=1.25.4 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from botocore<1.35.0,>=1.34.69->boto3==1.34.69->data_prep_toolkit==0.2.2.dev0) (2.2.2)\n", + "Requirement already satisfied: six>=1.5 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.35.0,>=1.34.69->boto3==1.34.69->data_prep_toolkit==0.2.2.dev0) (1.16.0)\n", + "Using cached argparse-1.4.0-py2.py3-none-any.whl (23 kB)\n", + "Building wheels for collected packages: data_prep_toolkit\n", + " Building editable for data_prep_toolkit (pyproject.toml): started\n", + " Building editable for data_prep_toolkit (pyproject.toml): finished with status 'done'\n", + " Created wheel for data_prep_toolkit: filename=data_prep_toolkit-0.2.2.dev0-0.editable-py3-none-any.whl size=2269 sha256=5abf7df19a437f63a327ddb160b27c2383d0094a831298eff28e89ad4edcad45\n", + " Stored in directory: /private/var/folders/f7/736gtk714sg36xlcqnjx16xw0000gn/T/pip-ephem-wheel-cache-5fkx2tua/wheels/eb/18/bc/2b7f918df1f95453f58d1ccde4ae0ea1bfd48079af6ea338f8\n", + "Successfully built data_prep_toolkit\n", + "Installing collected packages: argparse, data_prep_toolkit\n", + "Successfully installed argparse-1.4.0 data_prep_toolkit-0.2.2.dev0\n", + "Done installing source from ../../../../data-processing-lib/python into venv\n", + "Installed source from Python processing library for /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/bin/python\n", + "Installing from pyproject.toml\n", + "Obtaining file:///Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python\n", + " Installing build dependencies: started\n", + " Installing build dependencies: finished with status 'done'\n", + " Checking if build backend supports build_editable: started\n", + " Checking if build backend supports build_editable: finished with status 'done'\n", + " Getting requirements to build editable: started\n", + " Getting requirements to build editable: finished with status 'done'\n", + " Preparing editable metadata (pyproject.toml): started\n", + " Preparing editable metadata (pyproject.toml): finished with status 'done'\n", + "Collecting data-prep-toolkit==0.2.1.dev0 (from dpk_syntactic_concept_extractor_transform_python==1.0.0)\n", + " Using cached data_prep_toolkit-0.2.1.dev0-py3-none-any.whl.metadata (1.9 kB)\n", + "Requirement already satisfied: parameterized in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (0.9.0)\n", + "Requirement already satisfied: pandas in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (2.2.2)\n", + "Requirement already satisfied: aiolimiter==1.1.0 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (1.1.0)\n", + "Requirement already satisfied: altair==5.3.0 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (5.3.0)\n", + "Requirement already satisfied: annotated-types==0.7.0 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (0.7.0)\n", + "Requirement already satisfied: anyio==4.4.0 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (4.4.0)\n", + "Requirement already satisfied: appnope==0.1.4 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (0.1.4)\n", + "Requirement already satisfied: asttokens==2.4.1 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (2.4.1)\n", + "Requirement already satisfied: attrs==23.2.0 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (23.2.0)\n", + "Requirement already satisfied: blinker==1.8.2 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (1.8.2)\n", + "Requirement already satisfied: cachetools==5.3.3 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (5.3.3)\n", + "Requirement already satisfied: certifi==2024.6.2 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (2024.6.2)\n", + "Requirement already satisfied: charset-normalizer==3.3.2 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (3.3.2)\n", + "Requirement already satisfied: click==8.1.7 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (8.1.7)\n", + "Requirement already satisfied: comm==0.2.2 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (0.2.2)\n", + "Requirement already satisfied: contourpy==1.2.1 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (1.2.1)\n", + "Requirement already satisfied: cycler==0.12.1 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (0.12.1)\n", + "Requirement already satisfied: debugpy==1.8.1 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (1.8.1)\n", + "Requirement already satisfied: decorator==5.1.1 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (5.1.1)\n", + "Requirement already satisfied: Deprecated==1.2.14 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (1.2.14)\n", + "Requirement already satisfied: executing==2.0.1 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (2.0.1)\n", + "Requirement already satisfied: fonttools==4.53.0 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (4.53.0)\n", + "Requirement already satisfied: gitdb==4.0.11 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (4.0.11)\n", + "Requirement already satisfied: GitPython==3.1.43 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (3.1.43)\n", + "Requirement already satisfied: h11==0.14.0 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (0.14.0)\n", + "Requirement already satisfied: htbuilder==0.6.2 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (0.6.2)\n", + "Requirement already satisfied: httpcore==1.0.5 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (1.0.5)\n", + "Requirement already satisfied: httpx==0.27.0 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (0.27.0)\n", + "Requirement already satisfied: httpx-sse==0.4.0 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (0.4.0)\n", + "Requirement already satisfied: ibm-generative-ai==3.0.0 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (3.0.0)\n", + "Requirement already satisfied: idna==3.7 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (3.7)\n", + "Requirement already satisfied: ipykernel==6.29.4 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (6.29.4)\n", + "Requirement already satisfied: ipython==8.25.0 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (8.25.0)\n", + "Requirement already satisfied: jedi==0.19.1 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (0.19.1)\n", + "Requirement already satisfied: Jinja2==3.1.4 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (3.1.4)\n", + "Requirement already satisfied: jsonschema==4.22.0 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (4.22.0)\n", + "Requirement already satisfied: jsonschema-specifications==2023.12.1 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (2023.12.1)\n", + "Requirement already satisfied: jupyter-client==8.6.2 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (8.6.2)\n", + "Requirement already satisfied: jupyter-core==5.7.2 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (5.7.2)\n", + "Requirement already satisfied: kiwisolver==1.4.5 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (1.4.5)\n", + "Requirement already satisfied: markdown-it-py==3.0.0 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (3.0.0)\n", + "Requirement already satisfied: MarkupSafe==2.1.5 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (2.1.5)\n", + "Requirement already satisfied: matplotlib==3.9.0 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (3.9.0)\n", + "Requirement already satisfied: matplotlib-inline==0.1.7 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (0.1.7)\n", + "Requirement already satisfied: mdurl==0.1.2 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (0.1.2)\n", + "Requirement already satisfied: more-itertools==10.3.0 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (10.3.0)\n", + "Requirement already satisfied: nest-asyncio==1.6.0 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (1.6.0)\n", + "Requirement already satisfied: networkx==3.3 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (3.3)\n", + "Requirement already satisfied: numpy==1.26.4 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (1.26.4)\n", + "Requirement already satisfied: packaging==24.0 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (24.0)\n", + "Requirement already satisfied: parso==0.8.4 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (0.8.4)\n", + "Requirement already satisfied: pexpect==4.9.0 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (4.9.0)\n", + "Requirement already satisfied: pillow==10.3.0 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (10.3.0)\n", + "Requirement already satisfied: platformdirs==4.2.2 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (4.2.2)\n", + "Requirement already satisfied: prompt-toolkit==3.0.45 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (3.0.45)\n", + "Requirement already satisfied: protobuf==5.27.2 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (5.27.2)\n", + "Requirement already satisfied: psutil==5.9.8 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (5.9.8)\n", + "Requirement already satisfied: ptyprocess==0.7.0 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (0.7.0)\n", + "Requirement already satisfied: pure-eval==0.2.2 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (0.2.2)\n", + "Requirement already satisfied: pyarrow==16.1.0 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (16.1.0)\n", + "Requirement already satisfied: pydantic==2.7.4 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (2.7.4)\n", + "Requirement already satisfied: pydantic-core==2.18.4 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (2.18.4)\n", + "Requirement already satisfied: pydeck==0.9.1 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (0.9.1)\n", + "Requirement already satisfied: Pygments==2.18.0 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (2.18.0)\n", + "Requirement already satisfied: pyparsing==3.1.2 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (3.1.2)\n", + "Requirement already satisfied: python-dateutil==2.9.0.post0 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (2.9.0.post0)\n", + "Requirement already satisfied: pytz==2024.1 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (2024.1)\n", + "Requirement already satisfied: pyzmq==26.0.3 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (26.0.3)\n", + "Requirement already satisfied: referencing==0.35.1 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (0.35.1)\n", + "Requirement already satisfied: regex==2024.5.15 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (2024.5.15)\n", + "Requirement already satisfied: requests==2.32.3 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (2.32.3)\n", + "Requirement already satisfied: rich==13.7.1 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (13.7.1)\n", + "Requirement already satisfied: rpds-py==0.18.1 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (0.18.1)\n", + "Requirement already satisfied: seaborn==0.13.2 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (0.13.2)\n", + "Requirement already satisfied: six==1.16.0 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (1.16.0)\n", + "Requirement already satisfied: smmap==5.0.1 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (5.0.1)\n", + "Requirement already satisfied: sniffio==1.3.1 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (1.3.1)\n", + "Requirement already satisfied: st-annotated-text==4.0.1 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (4.0.1)\n", + "Requirement already satisfied: stack-data==0.6.3 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (0.6.3)\n", + "Requirement already satisfied: streamlit==1.36.0 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (1.36.0)\n", + "Requirement already satisfied: tenacity==8.4.2 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (8.4.2)\n", + "Requirement already satisfied: toml==0.10.2 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (0.10.2)\n", + "Requirement already satisfied: toolz==0.12.1 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (0.12.1)\n", + "Requirement already satisfied: tornado==6.4 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (6.4)\n", + "Requirement already satisfied: traitlets==5.14.3 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (5.14.3)\n", + "Requirement already satisfied: tree-sitter==0.21.3 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (0.21.3)\n", + "Requirement already satisfied: tree-sitter-cpp==0.22.1 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (0.22.1)\n", + "Requirement already satisfied: tree-sitter-java==0.21.0 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (0.21.0)\n", + "Requirement already satisfied: tree-sitter-languages==1.10.2 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (1.10.2)\n", + "Requirement already satisfied: tree-sitter-php==0.22.5 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (0.22.5)\n", + "Requirement already satisfied: typing-extensions==4.12.2 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (4.12.2)\n", + "Requirement already satisfied: tzdata==2024.1 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (2024.1)\n", + "Requirement already satisfied: urllib3==2.2.2 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (2.2.2)\n", + "Requirement already satisfied: wcwidth==0.2.13 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (0.2.13)\n", + "Requirement already satisfied: wrapt==1.16.0 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from dpk_syntactic_concept_extractor_transform_python==1.0.0) (1.16.0)\n", + "Requirement already satisfied: boto3==1.34.69 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from data-prep-toolkit==0.2.1.dev0->dpk_syntactic_concept_extractor_transform_python==1.0.0) (1.34.69)\n", + "Collecting argparse (from data-prep-toolkit==0.2.1.dev0->dpk_syntactic_concept_extractor_transform_python==1.0.0)\n", + " Using cached argparse-1.4.0-py2.py3-none-any.whl.metadata (2.8 kB)\n", + "Requirement already satisfied: mmh3 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from data-prep-toolkit==0.2.1.dev0->dpk_syntactic_concept_extractor_transform_python==1.0.0) (5.0.1)\n", + "Requirement already satisfied: botocore<1.35.0,>=1.34.69 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit==0.2.1.dev0->dpk_syntactic_concept_extractor_transform_python==1.0.0) (1.34.162)\n", + "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit==0.2.1.dev0->dpk_syntactic_concept_extractor_transform_python==1.0.0) (1.0.1)\n", + "Requirement already satisfied: s3transfer<0.11.0,>=0.10.0 in /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/venv/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit==0.2.1.dev0->dpk_syntactic_concept_extractor_transform_python==1.0.0) (0.10.2)\n", + "Using cached data_prep_toolkit-0.2.1.dev0-py3-none-any.whl (63 kB)\n", + "Using cached argparse-1.4.0-py2.py3-none-any.whl (23 kB)\n", + "Building wheels for collected packages: dpk_syntactic_concept_extractor_transform_python\n", + " Building editable for dpk_syntactic_concept_extractor_transform_python (pyproject.toml): started\n", + " Building editable for dpk_syntactic_concept_extractor_transform_python (pyproject.toml): finished with status 'done'\n", + " Created wheel for dpk_syntactic_concept_extractor_transform_python: filename=dpk_syntactic_concept_extractor_transform_python-1.0.0-0.editable-py3-none-any.whl size=3593 sha256=510f2fb9ae185f2fcfa6010c7e0be48b529dcbc230d91aa22a089c3e6970c686\n", + " Stored in directory: /private/var/folders/f7/736gtk714sg36xlcqnjx16xw0000gn/T/pip-ephem-wheel-cache-1u2gu5sh/wheels/05/e5/80/5ba70e3ece1b81ec352ba240e5a1e67a07c1d70e012b20ab06\n", + "Successfully built dpk_syntactic_concept_extractor_transform_python\n", + "Installing collected packages: argparse, data-prep-toolkit, dpk_syntactic_concept_extractor_transform_python\n", + " Attempting uninstall: data-prep-toolkit\n", + " Found existing installation: data_prep_toolkit 0.2.2.dev0\n", + " Uninstalling data_prep_toolkit-0.2.2.dev0:\n", + " Successfully uninstalled data_prep_toolkit-0.2.2.dev0\n", + " Attempting uninstall: dpk_syntactic_concept_extractor_transform_python\n", + " Found existing installation: dpk_syntactic_concept_extractor_transform_python 1.0.0\n", + " Uninstalling dpk_syntactic_concept_extractor_transform_python-1.0.0:\n", + " Successfully uninstalled dpk_syntactic_concept_extractor_transform_python-1.0.0\n", + "Successfully installed argparse-1.4.0 data-prep-toolkit-0.2.1.dev0 dpk_syntactic_concept_extractor_transform_python-1.0.0\n", + "Process completed successfully.\n", + "Changed directory to: /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python\n", + "/Library/Developer/CommandLineTools/usr/bin/make RUN_FILE=syntactic_concept_extractor_local.py .transforms.run-src-file\n", + "source venv/bin/activate;\t\\\n", + "\tcd src;\t\t\t\t\\\n", + "\tpython syntactic_concept_extractor_local.py \"\"\n", + "Loaded dictionary: {'input': 'multi-package.parquet', 'contents': 'Contents', 'language': 'Language'}\n", + "Syntactic constructs extraction started\n", + "Checking directory: /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python/src\n", + "Checking directory: /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/python\n", + "Checking directory: /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor\n", + "Checking directory: /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/transforms/code\n", + "Checking directory: /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/transforms\n", + "Checking directory: /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit\n", + "Project root found: /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit\n", + "Bindings path: /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/input/tree-sitter-bindings/mach-arm64\n", + "input table: pyarrow.Table\n", + "Language: string\n", + "Category: string\n", + "Library: string\n", + "Contents: string\n", + "Weight: double\n", + "----\n", + "Language: [[\"C\",\"C\",\"C\",\"C\",\"C\",...,\"Go\",\"Go\",\"Go\",\"Go\",\"Go\"]]\n", + "Category: [[\"Algorithms and Data Structures\",\"Algorithms and Data Structures\",\"Algorithms and Data Structures\",\"Algorithms and Data Structures\",\"Algorithms and Data Structures\",...,\"Testing\",\"Web Development\",\"Web Development\",\"Web Development\",\"Web Development\"]]\n", + "Library: [[\"dotnet-mgcb-compute, Akade.IndexedSet, Akka.DistributedData\",\"dotnet-mgcb-compute\",\"Akade.IndexedSet\",\"dotnet-mgcb-compute, QuantConnect.Algorithm.CSharp\",\"QuantConnect.Algorithm.CSharp\",...,\"github.com/stretchr/testify, gopkg.in/check.v1\",\"github.com/gorilla/sessions, github.com/labstack/echo/v4\",\"github.com/labstack/echo/v4, github.com/gorilla/sessions, github.com/gin-gonic/gin\",\"github.com/labstack/echo/v4, github.com/gin-gonic/gin\",\"github.com/gin-gonic/gin, github.com/labstack/echo/v4, github.com/gorilla/sessions\"]]\n", + "Contents: [[\"#include \"dotnet-mgcb-compute.h\"\n", + "#include \"Akade.IndexedSet.h\"\n", + "#include \"Akka.DistributedData.h\"\n", + "// Function with parameters\n", + "void baz1(int x) {\n", + " /* TODO: Implement this */\n", + "}\n", + "int main() {\n", + " baz1(10);\n", + " return 0;\n", + "}\n", + "\",\"#include \"dotnet-mgcb-compute.h\"\n", + "// Function example\n", + "void foo2() {\n", + " /* TODO: Add your code here */\n", + "}\n", + "int main() {\n", + " foo2();\n", + " return 0;\n", + "}\n", + "\",\"#include \"Akade.IndexedSet.h\"\n", + "// Function with return\n", + "int quux3() {\n", + " return 42;\n", + "}\n", + "int main() {\n", + " int result = quux3();\n", + " return 0;\n", + "}\n", + "\",\"#include \"dotnet-mgcb-compute.h\"\n", + "#include \"QuantConnect.Algorithm.CSharp.h\"\n", + "// Function with parameters\n", + "void baz4(int x) {\n", + " /* TODO: Implement this */\n", + "}\n", + "int main() {\n", + " baz4(10);\n", + " return 0;\n", + "}\n", + "\",\"#include \"QuantConnect.Algorithm.CSharp.h\"\n", + "// Function with return\n", + "int quux5() {\n", + " return 42;\n", + "}\n", + "int main() {\n", + " int result = quux5();\n", + " return 0;\n", + "}\n", + "\",...,\"import \"github.com/stretchr/testify\"\n", + "import \"gopkg.in/check.v1\"\n", + "// Another example function\n", + "func bar396() int {\n", + " return 0;\n", + "}\n", + "\",\"import \"github.com/gorilla/sessions\"\n", + "import \"github.com/labstack/echo/v4\"\n", + "// Function with parameters\n", + "func baz397(x int) {\n", + " /* TODO: Implement this */\n", + "}\n", + "\",\"import \"github.com/labstack/echo/v4\"\n", + "import \"github.com/gorilla/sessions\"\n", + "import \"github.com/gin-gonic/gin\"\n", + "// Function example\n", + "func foo398() {\n", + " /* TODO: Add your code here */\n", + "}\n", + "\",\"import \"github.com/labstack/echo/v4\"\n", + "import \"github.com/gin-gonic/gin\"\n", + "// Function with parameters\n", + "func baz399(x int) {\n", + " /* TODO: Implement this */\n", + "}\n", + "\",\"import \"github.com/gin-gonic/gin\"\n", + "import \"github.com/labstack/echo/v4\"\n", + "import \"github.com/gorilla/sessions\"\n", + "// Function with return\n", + "func quux400() int {\n", + " return 42;\n", + "}\n", + "\"]]\n", + "Weight: [[0.0015624999999999999,0.0005208333333333334,0.0005208333333333334,0.0010416666666666669,0.0005208333333333334,...,0.0009259259259259261,0.0008333333333333334,0.0012500000000000002,0.0008333333333333334,0.0012500000000000002]]\n", + "other_val: 0\n", + "tranforming the the input dataframe\n", + "uni-algo\n", + "tries\n", + "uni-algo\n", + "tries\n", + "dynamic_bitset\n", + "algorithm\n", + "tries\n", + "clangd\n", + "clang-tidy\n", + "cquery\n", + "clangd\n", + "cppcheck\n", + "cquery\n", + "cppcheck\n", + "clangd\n", + "cquery\n", + "cppcheck\n", + "clang-tidy\n", + "cppcheck\n", + "clang-tidy\n", + "cquery\n", + "armadillo\n", + "dlib\n", + "oneapi/dal\n", + "blitz\n", + "armadillo\n", + "armadillo\n", + "blitz\n", + "oneapi/dal\n", + "blitz\n", + "oneapi/dal\n", + "boost.serialization\n", + "frozen\n", + "cppcodec\n", + "glaze\n", + "cppcodec\n", + "frozen\n", + "boost.serialization\n", + "cppcodec\n", + "frozen\n", + "boost.serialization\n", + "glaze\n", + "sqlite\n", + "libpqxx\n", + "clickhouse\n", + "sqlite\n", + "leveldb\n", + "clickhouse\n", + "sqlite\n", + "llfio\n", + "tinydir\n", + "llfio\n", + "tinydir\n", + "filesystem\n", + "tinydir\n", + "llfio\n", + "llfio\n", + "filesystem\n", + "tinydir\n", + "wxwidgets\n", + "cegui\n", + "nanogui\n", + "gtk\n", + "cegui\n", + "wxwidgets\n", + "gtk\n", + "cegui\n", + "nanogui\n", + "wxwidgets\n", + "cegui\n", + "wxwidgets\n", + "workflow\n", + "taskflow\n", + "workflow\n", + "libthrift\n", + "taskflow\n", + "infra\n", + "libthrift\n", + "infra\n", + "libthrift\n", + "libthrift\n", + "spdlog\n", + "reckless\n", + "spdlog\n", + "boost.log\n", + "spdlog\n", + "glog\n", + "boost.log\n", + "glog\n", + "spdlog\n", + "reckless\n", + "linalg\n", + "blaze\n", + "cnl\n", + "cnl\n", + "eigen\n", + "eigen\n", + "blaze\n", + "eigen\n", + "cnl\n", + "cnl\n", + "linalg\n", + "eigen\n", + "azmq\n", + "zmq\n", + "azmq\n", + "boost.asio\n", + "cpp-netlib\n", + "boost.asio\n", + "time\n", + "thread-pool\n", + "concurrencpp\n", + "time\n", + "chrono\n", + "concurrencpp\n", + "concurrencpp\n", + "time\n", + "chrono\n", + "thread-pool\n", + "libgcrypt\n", + "digestpp\n", + "libgcrypt\n", + "digestpp\n", + "libressl\n", + "libgcrypt\n", + "digestpp\n", + "libressl\n", + "digestpp\n", + "libgcrypt\n", + "cpputest\n", + "ctest\n", + "cpputest\n", + "ctest\n", + "benchmark\n", + "ctest\n", + "boost.test\n", + "cpputest\n", + "cpputest\n", + "libfv\n", + "jwt-cpp\n", + "jwt-cpp\n", + "libfv\n", + "libonion\n", + "libfv\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "list index out of range\n", + "1\n", + "\n", + "output table has 400 rows and 6 columns\n", + "output metadata : {'source_documents': 5, 'result_documents': 6}\n", + "Table 0 saved to /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/transforms/code/syntactic_concept_extractor/output/uast_table_part_0.parquet\n", + "Process completed successfully.\n" + ] + }, + { + "data": { + "text/plain": [ + "'/Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/transforms/code/semantic_profiler/python/test-data/input/uast_table_part_0.parquet'" + ] + }, + "execution_count": 63, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# runnning the syntactic_concept_extractor transform\n", + "run_make_command(sce_code_dir, ['make', 'venv'])\n", + "run_make_command(sce_code_dir, ['make', 'run-local-sample'])\n", + "\n", + "# Copy the output of the syntactic_concept_extractor transform to the input folder of the semantic_profiler\n", + "output_pq = '/uast_table_part_0.parquet'\n", + "shutil.copy2(sce_output_dir + output_pq, sp_input_dir + output_pq)" + ] + }, + { + "cell_type": "code", + "execution_count": 64, + "id": "b6458fa0-e459-47b9-951b-a209e59c4481", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Changed directory to: /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/transforms/code/semantic_profiler/python\n", + "python -m venv venv\n", + "Requirement already satisfied: pip in ./venv/lib/python3.11/site-packages (24.2)\n", + "Requirement already satisfied: wheel in ./venv/lib/python3.11/site-packages (0.44.0)\n", + "Requirement already satisfied: pytest in ./venv/lib/python3.11/site-packages (8.3.3)\n", + "Requirement already satisfied: pytest-cov in ./venv/lib/python3.11/site-packages (5.0.0)\n", + "Requirement already satisfied: iniconfig in ./venv/lib/python3.11/site-packages (from pytest) (2.0.0)\n", + "Requirement already satisfied: packaging in ./venv/lib/python3.11/site-packages (from pytest) (24.1)\n", + "Requirement already satisfied: pluggy<2,>=1.5 in ./venv/lib/python3.11/site-packages (from pytest) (1.5.0)\n", + "Requirement already satisfied: coverage>=5.2.1 in ./venv/lib/python3.11/site-packages (from coverage[toml]>=5.2.1->pytest-cov) (7.6.1)\n", + "Installing Python data processing library source to existing venv\n", + "pip uninstall -y data-prep-toolkit \n", + "Found existing installation: data_prep_toolkit 0.2.1.dev0\n", + "Uninstalling data_prep_toolkit-0.2.1.dev0:\n", + " Successfully uninstalled data_prep_toolkit-0.2.1.dev0\n", + "Begin installing source from ../../../../data-processing-lib/python into venv\n", + "Obtaining file:///Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/data-processing-lib/python\n", + " Installing build dependencies: started\n", + " Installing build dependencies: finished with status 'done'\n", + " Checking if build backend supports build_editable: started\n", + " Checking if build backend supports build_editable: finished with status 'done'\n", + " Getting requirements to build editable: started\n", + " Getting requirements to build editable: finished with status 'done'\n", + " Preparing editable metadata (pyproject.toml): started\n", + " Preparing editable metadata (pyproject.toml): finished with status 'done'\n", + "Requirement already satisfied: numpy<1.29.0 in ./venv/lib/python3.11/site-packages (from data_prep_toolkit==0.2.2.dev0) (1.26.4)\n", + "Requirement already satisfied: pyarrow==16.1.0 in ./venv/lib/python3.11/site-packages (from data_prep_toolkit==0.2.2.dev0) (16.1.0)\n", + "Requirement already satisfied: boto3==1.34.69 in ./venv/lib/python3.11/site-packages (from data_prep_toolkit==0.2.2.dev0) (1.34.69)\n", + "Collecting argparse (from data_prep_toolkit==0.2.2.dev0)\n", + " Using cached argparse-1.4.0-py2.py3-none-any.whl.metadata (2.8 kB)\n", + "Requirement already satisfied: mmh3 in ./venv/lib/python3.11/site-packages (from data_prep_toolkit==0.2.2.dev0) (5.0.1)\n", + "Requirement already satisfied: botocore<1.35.0,>=1.34.69 in ./venv/lib/python3.11/site-packages (from boto3==1.34.69->data_prep_toolkit==0.2.2.dev0) (1.34.162)\n", + "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in ./venv/lib/python3.11/site-packages (from boto3==1.34.69->data_prep_toolkit==0.2.2.dev0) (1.0.1)\n", + "Requirement already satisfied: s3transfer<0.11.0,>=0.10.0 in ./venv/lib/python3.11/site-packages (from boto3==1.34.69->data_prep_toolkit==0.2.2.dev0) (0.10.2)\n", + "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in ./venv/lib/python3.11/site-packages (from botocore<1.35.0,>=1.34.69->boto3==1.34.69->data_prep_toolkit==0.2.2.dev0) (2.9.0.post0)\n", + "Requirement already satisfied: urllib3!=2.2.0,<3,>=1.25.4 in ./venv/lib/python3.11/site-packages (from botocore<1.35.0,>=1.34.69->boto3==1.34.69->data_prep_toolkit==0.2.2.dev0) (2.2.3)\n", + "Requirement already satisfied: six>=1.5 in ./venv/lib/python3.11/site-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.35.0,>=1.34.69->boto3==1.34.69->data_prep_toolkit==0.2.2.dev0) (1.16.0)\n", + "Using cached argparse-1.4.0-py2.py3-none-any.whl (23 kB)\n", + "Building wheels for collected packages: data_prep_toolkit\n", + " Building editable for data_prep_toolkit (pyproject.toml): started\n", + " Building editable for data_prep_toolkit (pyproject.toml): finished with status 'done'\n", + " Created wheel for data_prep_toolkit: filename=data_prep_toolkit-0.2.2.dev0-0.editable-py3-none-any.whl size=2269 sha256=1e48b0e59e63118611e05d6b17d1c64ad91cb4a2bc3b1d517b608040d218a3ff\n", + " Stored in directory: /private/var/folders/f7/736gtk714sg36xlcqnjx16xw0000gn/T/pip-ephem-wheel-cache-7425eo0g/wheels/eb/18/bc/2b7f918df1f95453f58d1ccde4ae0ea1bfd48079af6ea338f8\n", + "Successfully built data_prep_toolkit\n", + "Installing collected packages: argparse, data_prep_toolkit\n", + "Successfully installed argparse-1.4.0 data_prep_toolkit-0.2.2.dev0\n", + "Done installing source from ../../../../data-processing-lib/python into venv\n", + "Installed source from Python processing library for /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/transforms/code/semantic_profiler/python/venv/bin/python\n", + "Installing from pyproject.toml\n", + "Obtaining file:///Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/transforms/code/semantic_profiler/python\n", + " Installing build dependencies: started\n", + " Installing build dependencies: finished with status 'done'\n", + " Checking if build backend supports build_editable: started\n", + " Checking if build backend supports build_editable: finished with status 'done'\n", + " Getting requirements to build editable: started\n", + " Getting requirements to build editable: finished with status 'done'\n", + " Preparing editable metadata (pyproject.toml): started\n", + " Preparing editable metadata (pyproject.toml): finished with status 'done'\n", + "Collecting data-prep-toolkit==0.2.1.dev0 (from dpk_sp_transform_python==0.2.1.dev0)\n", + " Using cached data_prep_toolkit-0.2.1.dev0-py3-none-any.whl.metadata (1.9 kB)\n", + "Requirement already satisfied: pyarrow==16.1.0 in ./venv/lib/python3.11/site-packages (from data-prep-toolkit==0.2.1.dev0->dpk_sp_transform_python==0.2.1.dev0) (16.1.0)\n", + "Requirement already satisfied: boto3==1.34.69 in ./venv/lib/python3.11/site-packages (from data-prep-toolkit==0.2.1.dev0->dpk_sp_transform_python==0.2.1.dev0) (1.34.69)\n", + "Collecting argparse (from data-prep-toolkit==0.2.1.dev0->dpk_sp_transform_python==0.2.1.dev0)\n", + " Using cached argparse-1.4.0-py2.py3-none-any.whl.metadata (2.8 kB)\n", + "Requirement already satisfied: mmh3 in ./venv/lib/python3.11/site-packages (from data-prep-toolkit==0.2.1.dev0->dpk_sp_transform_python==0.2.1.dev0) (5.0.1)\n", + "Requirement already satisfied: botocore<1.35.0,>=1.34.69 in ./venv/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit==0.2.1.dev0->dpk_sp_transform_python==0.2.1.dev0) (1.34.162)\n", + "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in ./venv/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit==0.2.1.dev0->dpk_sp_transform_python==0.2.1.dev0) (1.0.1)\n", + "Requirement already satisfied: s3transfer<0.11.0,>=0.10.0 in ./venv/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit==0.2.1.dev0->dpk_sp_transform_python==0.2.1.dev0) (0.10.2)\n", + "Requirement already satisfied: numpy>=1.16.6 in ./venv/lib/python3.11/site-packages (from pyarrow==16.1.0->data-prep-toolkit==0.2.1.dev0->dpk_sp_transform_python==0.2.1.dev0) (1.26.4)\n", + "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in ./venv/lib/python3.11/site-packages (from botocore<1.35.0,>=1.34.69->boto3==1.34.69->data-prep-toolkit==0.2.1.dev0->dpk_sp_transform_python==0.2.1.dev0) (2.9.0.post0)\n", + "Requirement already satisfied: urllib3!=2.2.0,<3,>=1.25.4 in ./venv/lib/python3.11/site-packages (from botocore<1.35.0,>=1.34.69->boto3==1.34.69->data-prep-toolkit==0.2.1.dev0->dpk_sp_transform_python==0.2.1.dev0) (2.2.3)\n", + "Requirement already satisfied: six>=1.5 in ./venv/lib/python3.11/site-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.35.0,>=1.34.69->boto3==1.34.69->data-prep-toolkit==0.2.1.dev0->dpk_sp_transform_python==0.2.1.dev0) (1.16.0)\n", + "Using cached data_prep_toolkit-0.2.1.dev0-py3-none-any.whl (63 kB)\n", + "Using cached argparse-1.4.0-py2.py3-none-any.whl (23 kB)\n", + "Building wheels for collected packages: dpk_sp_transform_python\n", + " Building editable for dpk_sp_transform_python (pyproject.toml): started\n", + " Building editable for dpk_sp_transform_python (pyproject.toml): finished with status 'done'\n", + " Created wheel for dpk_sp_transform_python: filename=dpk_sp_transform_python-0.2.1.dev0-0.editable-py3-none-any.whl size=3029 sha256=b974ec3554a35c3eed9404259858fe9e0f32bae721292b2720ba1d0f1554c09a\n", + " Stored in directory: /private/var/folders/f7/736gtk714sg36xlcqnjx16xw0000gn/T/pip-ephem-wheel-cache-tz9u49oy/wheels/2b/2b/a9/43d967436e095f83fb907da3c3f1e509b63e5e34561366fcc5\n", + "Successfully built dpk_sp_transform_python\n", + "Installing collected packages: argparse, data-prep-toolkit, dpk_sp_transform_python\n", + " Attempting uninstall: data-prep-toolkit\n", + " Found existing installation: data_prep_toolkit 0.2.2.dev0\n", + " Uninstalling data_prep_toolkit-0.2.2.dev0:\n", + " Successfully uninstalled data_prep_toolkit-0.2.2.dev0\n", + " Attempting uninstall: dpk_sp_transform_python\n", + " Found existing installation: dpk_sp_transform_python 0.2.1.dev0\n", + " Uninstalling dpk_sp_transform_python-0.2.1.dev0:\n", + " Successfully uninstalled dpk_sp_transform_python-0.2.1.dev0\n", + "Successfully installed argparse-1.4.0 data-prep-toolkit-0.2.1.dev0 dpk_sp_transform_python-0.2.1.dev0\n", + "Process completed successfully.\n", + "Changed directory to: /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/transforms/code/semantic_profiler/python\n", + "/Library/Developer/CommandLineTools/usr/bin/make RUN_FILE=sp_local_python.py .transforms.run-src-file\n", + "source venv/bin/activate;\t\\\n", + "\tcd src;\t\t\t\t\\\n", + "\tpython sp_local_python.py \"\"\n", + "Process completed successfully.\n" + ] + }, + { + "data": { + "text/plain": [ + "'/Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/transforms/code/higher_order_syntactic_profiler/python/test-data/input/uast_table_part_0.parquet'" + ] + }, + "execution_count": 64, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# runnning the semantic_profiler transform\n", + "run_make_command(sp_code_dir, ['make', 'venv'])\n", + "run_make_command(sp_code_dir, ['make', 'run-local-python-sample'])\n", + "\n", + "# Copy the output of the semantic_profiler transform to the input folder of the higher_order_syntactic_profiler\n", + "shutil.copy2(sp_output_dir + output_pq, hosp_input_dir + output_pq)" + ] + }, + { + "cell_type": "code", + "execution_count": 65, + "id": "9bedf5dd-92d6-4a83-abb5-f8439b80ee02", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Changed directory to: /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/transforms/code/higher_order_syntactic_profiler/python\n", + "python -m venv venv\n", + "Requirement already satisfied: pip in ./venv/lib/python3.11/site-packages (24.2)\n", + "Requirement already satisfied: wheel in ./venv/lib/python3.11/site-packages (0.44.0)\n", + "Requirement already satisfied: pytest in ./venv/lib/python3.11/site-packages (8.3.3)\n", + "Requirement already satisfied: pytest-cov in ./venv/lib/python3.11/site-packages (5.0.0)\n", + "Requirement already satisfied: iniconfig in ./venv/lib/python3.11/site-packages (from pytest) (2.0.0)\n", + "Requirement already satisfied: packaging in ./venv/lib/python3.11/site-packages (from pytest) (24.1)\n", + "Requirement already satisfied: pluggy<2,>=1.5 in ./venv/lib/python3.11/site-packages (from pytest) (1.5.0)\n", + "Requirement already satisfied: coverage>=5.2.1 in ./venv/lib/python3.11/site-packages (from coverage[toml]>=5.2.1->pytest-cov) (7.6.1)\n", + "Installing Python data processing library source to existing venv\n", + "pip uninstall -y data-prep-toolkit \n", + "Found existing installation: data_prep_toolkit 0.2.1.dev0\n", + "Uninstalling data_prep_toolkit-0.2.1.dev0:\n", + " Successfully uninstalled data_prep_toolkit-0.2.1.dev0\n", + "Begin installing source from ../../../../data-processing-lib/python into venv\n", + "Obtaining file:///Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/data-processing-lib/python\n", + " Installing build dependencies: started\n", + " Installing build dependencies: finished with status 'done'\n", + " Checking if build backend supports build_editable: started\n", + " Checking if build backend supports build_editable: finished with status 'done'\n", + " Getting requirements to build editable: started\n", + " Getting requirements to build editable: finished with status 'done'\n", + " Preparing editable metadata (pyproject.toml): started\n", + " Preparing editable metadata (pyproject.toml): finished with status 'done'\n", + "Requirement already satisfied: numpy<1.29.0 in ./venv/lib/python3.11/site-packages (from data_prep_toolkit==0.2.2.dev0) (1.26.4)\n", + "Requirement already satisfied: pyarrow==16.1.0 in ./venv/lib/python3.11/site-packages (from data_prep_toolkit==0.2.2.dev0) (16.1.0)\n", + "Requirement already satisfied: boto3==1.34.69 in ./venv/lib/python3.11/site-packages (from data_prep_toolkit==0.2.2.dev0) (1.34.69)\n", + "Collecting argparse (from data_prep_toolkit==0.2.2.dev0)\n", + " Using cached argparse-1.4.0-py2.py3-none-any.whl.metadata (2.8 kB)\n", + "Requirement already satisfied: mmh3 in ./venv/lib/python3.11/site-packages (from data_prep_toolkit==0.2.2.dev0) (5.0.1)\n", + "Requirement already satisfied: botocore<1.35.0,>=1.34.69 in ./venv/lib/python3.11/site-packages (from boto3==1.34.69->data_prep_toolkit==0.2.2.dev0) (1.34.162)\n", + "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in ./venv/lib/python3.11/site-packages (from boto3==1.34.69->data_prep_toolkit==0.2.2.dev0) (1.0.1)\n", + "Requirement already satisfied: s3transfer<0.11.0,>=0.10.0 in ./venv/lib/python3.11/site-packages (from boto3==1.34.69->data_prep_toolkit==0.2.2.dev0) (0.10.2)\n", + "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in ./venv/lib/python3.11/site-packages (from botocore<1.35.0,>=1.34.69->boto3==1.34.69->data_prep_toolkit==0.2.2.dev0) (2.9.0.post0)\n", + "Requirement already satisfied: urllib3!=2.2.0,<3,>=1.25.4 in ./venv/lib/python3.11/site-packages (from botocore<1.35.0,>=1.34.69->boto3==1.34.69->data_prep_toolkit==0.2.2.dev0) (2.2.3)\n", + "Requirement already satisfied: six>=1.5 in ./venv/lib/python3.11/site-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.35.0,>=1.34.69->boto3==1.34.69->data_prep_toolkit==0.2.2.dev0) (1.16.0)\n", + "Using cached argparse-1.4.0-py2.py3-none-any.whl (23 kB)\n", + "Building wheels for collected packages: data_prep_toolkit\n", + " Building editable for data_prep_toolkit (pyproject.toml): started\n", + " Building editable for data_prep_toolkit (pyproject.toml): finished with status 'done'\n", + " Created wheel for data_prep_toolkit: filename=data_prep_toolkit-0.2.2.dev0-0.editable-py3-none-any.whl size=2269 sha256=ca207188a4f4284ff51582aa6e5ad215097da2b72edce0a0ee5abf4b66c9753c\n", + " Stored in directory: /private/var/folders/f7/736gtk714sg36xlcqnjx16xw0000gn/T/pip-ephem-wheel-cache-o861wrsm/wheels/eb/18/bc/2b7f918df1f95453f58d1ccde4ae0ea1bfd48079af6ea338f8\n", + "Successfully built data_prep_toolkit\n", + "Installing collected packages: argparse, data_prep_toolkit\n", + "Successfully installed argparse-1.4.0 data_prep_toolkit-0.2.2.dev0\n", + "Done installing source from ../../../../data-processing-lib/python into venv\n", + "Installed source from Python processing library for /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/transforms/code/higher_order_syntactic_profiler/python/venv/bin/python\n", + "Installing from pyproject.toml\n", + "Obtaining file:///Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/transforms/code/higher_order_syntactic_profiler/python\n", + " Installing build dependencies: started\n", + " Installing build dependencies: finished with status 'done'\n", + " Checking if build backend supports build_editable: started\n", + " Checking if build backend supports build_editable: finished with status 'done'\n", + " Getting requirements to build editable: started\n", + " Getting requirements to build editable: finished with status 'done'\n", + " Preparing editable metadata (pyproject.toml): started\n", + " Preparing editable metadata (pyproject.toml): finished with status 'done'\n", + "Collecting data-prep-toolkit==0.2.1.dev0 (from dpk_hosp_transform_python==0.2.1.dev0)\n", + " Using cached data_prep_toolkit-0.2.1.dev0-py3-none-any.whl.metadata (1.9 kB)\n", + "Requirement already satisfied: networkx==3.0.0 in ./venv/lib/python3.11/site-packages (from dpk_hosp_transform_python==0.2.1.dev0) (3.0)\n", + "Requirement already satisfied: jinja2==3.1.2 in ./venv/lib/python3.11/site-packages (from dpk_hosp_transform_python==0.2.1.dev0) (3.1.2)\n", + "Requirement already satisfied: plotly==5.15.0 in ./venv/lib/python3.11/site-packages (from dpk_hosp_transform_python==0.2.1.dev0) (5.15.0)\n", + "Requirement already satisfied: matplotlib==3.9.0 in ./venv/lib/python3.11/site-packages (from dpk_hosp_transform_python==0.2.1.dev0) (3.9.0)\n", + "Requirement already satisfied: matplotlib-inline==0.1.7 in ./venv/lib/python3.11/site-packages (from dpk_hosp_transform_python==0.2.1.dev0) (0.1.7)\n", + "Requirement already satisfied: pyarrow==16.1.0 in ./venv/lib/python3.11/site-packages (from data-prep-toolkit==0.2.1.dev0->dpk_hosp_transform_python==0.2.1.dev0) (16.1.0)\n", + "Requirement already satisfied: boto3==1.34.69 in ./venv/lib/python3.11/site-packages (from data-prep-toolkit==0.2.1.dev0->dpk_hosp_transform_python==0.2.1.dev0) (1.34.69)\n", + "Collecting argparse (from data-prep-toolkit==0.2.1.dev0->dpk_hosp_transform_python==0.2.1.dev0)\n", + " Using cached argparse-1.4.0-py2.py3-none-any.whl.metadata (2.8 kB)\n", + "Requirement already satisfied: mmh3 in ./venv/lib/python3.11/site-packages (from data-prep-toolkit==0.2.1.dev0->dpk_hosp_transform_python==0.2.1.dev0) (5.0.1)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in ./venv/lib/python3.11/site-packages (from jinja2==3.1.2->dpk_hosp_transform_python==0.2.1.dev0) (2.1.5)\n", + "Requirement already satisfied: contourpy>=1.0.1 in ./venv/lib/python3.11/site-packages (from matplotlib==3.9.0->dpk_hosp_transform_python==0.2.1.dev0) (1.3.0)\n", + "Requirement already satisfied: cycler>=0.10 in ./venv/lib/python3.11/site-packages (from matplotlib==3.9.0->dpk_hosp_transform_python==0.2.1.dev0) (0.12.1)\n", + "Requirement already satisfied: fonttools>=4.22.0 in ./venv/lib/python3.11/site-packages (from matplotlib==3.9.0->dpk_hosp_transform_python==0.2.1.dev0) (4.54.1)\n", + "Requirement already satisfied: kiwisolver>=1.3.1 in ./venv/lib/python3.11/site-packages (from matplotlib==3.9.0->dpk_hosp_transform_python==0.2.1.dev0) (1.4.7)\n", + "Requirement already satisfied: numpy>=1.23 in ./venv/lib/python3.11/site-packages (from matplotlib==3.9.0->dpk_hosp_transform_python==0.2.1.dev0) (1.26.4)\n", + "Requirement already satisfied: packaging>=20.0 in ./venv/lib/python3.11/site-packages (from matplotlib==3.9.0->dpk_hosp_transform_python==0.2.1.dev0) (24.1)\n", + "Requirement already satisfied: pillow>=8 in ./venv/lib/python3.11/site-packages (from matplotlib==3.9.0->dpk_hosp_transform_python==0.2.1.dev0) (10.4.0)\n", + "Requirement already satisfied: pyparsing>=2.3.1 in ./venv/lib/python3.11/site-packages (from matplotlib==3.9.0->dpk_hosp_transform_python==0.2.1.dev0) (3.1.4)\n", + "Requirement already satisfied: python-dateutil>=2.7 in ./venv/lib/python3.11/site-packages (from matplotlib==3.9.0->dpk_hosp_transform_python==0.2.1.dev0) (2.9.0.post0)\n", + "Requirement already satisfied: traitlets in ./venv/lib/python3.11/site-packages (from matplotlib-inline==0.1.7->dpk_hosp_transform_python==0.2.1.dev0) (5.14.3)\n", + "Requirement already satisfied: tenacity>=6.2.0 in ./venv/lib/python3.11/site-packages (from plotly==5.15.0->dpk_hosp_transform_python==0.2.1.dev0) (9.0.0)\n", + "Requirement already satisfied: botocore<1.35.0,>=1.34.69 in ./venv/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit==0.2.1.dev0->dpk_hosp_transform_python==0.2.1.dev0) (1.34.162)\n", + "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in ./venv/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit==0.2.1.dev0->dpk_hosp_transform_python==0.2.1.dev0) (1.0.1)\n", + "Requirement already satisfied: s3transfer<0.11.0,>=0.10.0 in ./venv/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit==0.2.1.dev0->dpk_hosp_transform_python==0.2.1.dev0) (0.10.2)\n", + "Requirement already satisfied: six>=1.5 in ./venv/lib/python3.11/site-packages (from python-dateutil>=2.7->matplotlib==3.9.0->dpk_hosp_transform_python==0.2.1.dev0) (1.16.0)\n", + "Requirement already satisfied: urllib3!=2.2.0,<3,>=1.25.4 in ./venv/lib/python3.11/site-packages (from botocore<1.35.0,>=1.34.69->boto3==1.34.69->data-prep-toolkit==0.2.1.dev0->dpk_hosp_transform_python==0.2.1.dev0) (2.2.3)\n", + "Using cached data_prep_toolkit-0.2.1.dev0-py3-none-any.whl (63 kB)\n", + "Using cached argparse-1.4.0-py2.py3-none-any.whl (23 kB)\n", + "Building wheels for collected packages: dpk_hosp_transform_python\n", + " Building editable for dpk_hosp_transform_python (pyproject.toml): started\n", + " Building editable for dpk_hosp_transform_python (pyproject.toml): finished with status 'done'\n", + " Created wheel for dpk_hosp_transform_python: filename=dpk_hosp_transform_python-0.2.1.dev0-0.editable-py3-none-any.whl size=2998 sha256=3271dc1db830b824f1eaa39f145d1d27cdc76ad17ad5c80bdd29632bd33e9474\n", + " Stored in directory: /private/var/folders/f7/736gtk714sg36xlcqnjx16xw0000gn/T/pip-ephem-wheel-cache-44v7a13k/wheels/42/6a/97/42ad1db1552a59fdbe3c8110698b7fd0e6308328b1ce136aa7\n", + "Successfully built dpk_hosp_transform_python\n", + "Installing collected packages: argparse, data-prep-toolkit, dpk_hosp_transform_python\n", + " Attempting uninstall: data-prep-toolkit\n", + " Found existing installation: data_prep_toolkit 0.2.2.dev0\n", + " Uninstalling data_prep_toolkit-0.2.2.dev0:\n", + " Successfully uninstalled data_prep_toolkit-0.2.2.dev0\n", + " Attempting uninstall: dpk_hosp_transform_python\n", + " Found existing installation: dpk_hosp_transform_python 0.2.1.dev0\n", + " Uninstalling dpk_hosp_transform_python-0.2.1.dev0:\n", + " Successfully uninstalled dpk_hosp_transform_python-0.2.1.dev0\n", + "Successfully installed argparse-1.4.0 data-prep-toolkit-0.2.1.dev0 dpk_hosp_transform_python-0.2.1.dev0\n", + "Process completed successfully.\n", + "Changed directory to: /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/transforms/code/higher_order_syntactic_profiler/python\n", + "/Library/Developer/CommandLineTools/usr/bin/make RUN_FILE=hosp_local_python.py .transforms.run-src-file\n", + "source venv/bin/activate;\t\\\n", + "\tcd src;\t\t\t\t\\\n", + "\tpython hosp_local_python.py \"\"\n", + "HTML file generated: /Users/pankajthorat/challenges/7122-DP4genAI/os-data-prep-kit/v2-dpk/demo-dpk/data-prep-kit/transforms/code/higher_order_syntactic_profiler/python/src/output.html\n", + "Process completed successfully.\n" + ] + } + ], + "source": [ + "# runnning the higher_order_syntactic_profiler transform\n", + "run_make_command(hosp_code_dir, ['make', 'venv'])\n", + "run_make_command(hosp_code_dir, ['make', 'run-local-python-sample'])" + ] + }, + { + "cell_type": "code", + "execution_count": 66, + "id": "96166877-6c6d-4959-ba95-84a0716a0c5f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "
\n", + " \n", + "This report presents the detailed profiling report of the input dataset.
\n", + "