diff --git a/src/json2sql.egg-info/PKG-INFO b/src/json2sql.egg-info/PKG-INFO deleted file mode 100644 index eee09d9..0000000 --- a/src/json2sql.egg-info/PKG-INFO +++ /dev/null @@ -1,129 +0,0 @@ -Metadata-Version: 2.4 -Name: json2sql -Version: 0.1.0 -Summary: Convert JSON files/datasets to SQL INSERT statements -Author: Revenue Holdings -License-Expression: MIT -Keywords: json,sql,etl,data-conversion,cli -Classifier: Development Status :: 4 - Beta -Classifier: Intended Audience :: Developers -Classifier: Topic :: Database -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: 3.12 -Requires-Python: >=3.10 -Description-Content-Type: text/markdown -License-File: LICENSE -Requires-Dist: typer>=0.9.0 -Requires-Dist: rich>=13.0.0 -Provides-Extra: license -Requires-Dist: revenueholdings-license>=0.1.0; extra == "license" -Provides-Extra: dev -Requires-Dist: pytest>=7.0; extra == "dev" -Requires-Dist: pytest-cov; extra == "dev" -Dynamic: license-file - -# json2sql - -[![GitHub stars](https://img.shields.io/github/stars/Coding-Dev-Tools/json2sql?style=social)](https://github.com/Coding-Dev-Tools/json2sql/stargazers) - -Convert JSON files and datasets to SQL INSERT statements. Supports nested JSON, PostgreSQL, MySQL, and SQLite output dialects. - -[![PyPI](https://img.shields.io/pypi/v/json2sql)](https://pypi.org/project/json2sql/) -[![Python](https://img.shields.io/pypi/pyversions/json2sql)](https://pypi.org/project/json2sql/) -[![License](https://img.shields.io/pypi/l/json2sql)](https://github.com/Coding-Dev-Tools/json2sql/blob/main/LICENSE) - -**Why json2sql?** Moving data from JSON into a database should be one command, not a script you maintain. json2sql takes JSON files — flat or nested — and produces correct SQL INSERT statements in your dialect of choice. Nested objects are automatically flattened into relational tables. Arrays become multiple INSERT rows. Type inference handles strings, numbers, booleans, and nulls without configuration. Pipe data from stdin, specify the table name, and get clean SQL out. Zero dependencies beyond Python 3.10+ and the CLI. - -## Installation - -```bash -pip install json2sql -``` - -Or install directly from GitHub: - -```bash -pip install git+https://github.com/Coding-Dev-Tools/json2sql.git -``` - -## Quick Start - -```bash -# Basic usage — converts JSON to SQL INSERT statements -json2sql convert data.json - -# Specify output dialect -json2sql convert data.json --dialect postgres -json2sql convert data.json --dialect mysql -json2sql convert data.json --dialect sqlite - -# Specify output file -json2sql convert data.json -o output.sql - -# Specify table name -json2sql convert data.json --table users - -# Handle nested JSON (auto-flattens into relational tables) -json2sql convert nested_data.json --flatten -``` - -## Features - -- **Nested JSON support**: Automatically flattens nested objects into separate relational tables -- **Multi-dialect output**: PostgreSQL, MySQL, SQLite INSERT syntax -- **Array of objects**: Handles JSON arrays as multiple INSERT rows -- **Type inference**: Auto-detects strings, numbers, booleans, nulls -- **Pipe support**: Read from stdin for pipeline usage -- **Zero dependencies**: Only Python 3.10+ required (typer for CLI) - -## CI/CD Integration - -```bash -# Generate SQL as part of a data pipeline -cat data.json | json2sql convert --dialect postgres --table events > events.sql - -# Use in GitHub Actions to prepare test data -json2sql convert fixtures.json --dialect sqlite -o seed.sql -sqlite3 test.db < seed.sql -``` - -## Pricing - -json2sql is one of eight tools in the Revenue Holdings suite. One license covers all CLI tools. - -| Plan | Price | Best For | -|------|-------|----------| -| **Free** | $0 | Individual devs, OSS — CLI only, limited rows | -| **json2sql Individual** | **$9/mo** ($7 billed annually) | Professional devs — unlimited rows, batch processing | -| **Suite (all 8 tools)** | **$49/mo** ($39 billed annually) | Full Revenue Holdings toolkit — 40% savings | -| **Team** | **$79/mo** ($63 billed annually) | Up to 5 devs — API access, CI/CD integration, priority support | -| **Enterprise** | Custom | SSO, RBAC, compliance reports, dedicated support | - -🔹 **No lock-in**: CLI works fully offline on the free tier — no telemetry, no phone-home. -🔹 **Annual billing**: Save 20%. - -### Per-Tier Features - -| Feature | Free | json2sql | Suite | Team | Enterprise | -|---------|:----:|:--------:|:-----:|:----:|:----------:| -| CLI: convert, pipe | ✓ | ✓ | ✓ | ✓ | ✓ | -| Unlimited rows per conversion | — | ✓ | ✓ | ✓ | ✓ | -| Batch processing | — | ✓ | ✓ | ✓ | ✓ | -| Schema generation | — | ✓ | ✓ | ✓ | ✓ | -| API access | — | — | — | ✓ | ✓ | -| Compliance reports | — | — | — | — | ✓ | -| RBAC | — | — | — | — | ✓ | -| SSO / SAML / OIDC | — | — | — | — | ✓ | -| Priority support | Community | 24h | 24h | 8h | Dedicated | - ---- - -

- Part of Revenue Holdings — CLI tools built by autonomous AI. -

- -## License - -MIT diff --git a/src/json2sql.egg-info/SOURCES.txt b/src/json2sql.egg-info/SOURCES.txt deleted file mode 100644 index f67b394..0000000 --- a/src/json2sql.egg-info/SOURCES.txt +++ /dev/null @@ -1,14 +0,0 @@ -LICENSE -README.md -pyproject.toml -src/json2sql/__init__.py -src/json2sql/cli.py -src/json2sql/converter.py -src/json2sql/dialects.py -src/json2sql.egg-info/PKG-INFO -src/json2sql.egg-info/SOURCES.txt -src/json2sql.egg-info/dependency_links.txt -src/json2sql.egg-info/entry_points.txt -src/json2sql.egg-info/requires.txt -src/json2sql.egg-info/top_level.txt -tests/test_converter.py \ No newline at end of file diff --git a/src/json2sql.egg-info/dependency_links.txt b/src/json2sql.egg-info/dependency_links.txt deleted file mode 100644 index 8b13789..0000000 --- a/src/json2sql.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/src/json2sql.egg-info/entry_points.txt b/src/json2sql.egg-info/entry_points.txt deleted file mode 100644 index 899fe30..0000000 --- a/src/json2sql.egg-info/entry_points.txt +++ /dev/null @@ -1,2 +0,0 @@ -[console_scripts] -json2sql = json2sql.cli:app diff --git a/src/json2sql.egg-info/requires.txt b/src/json2sql.egg-info/requires.txt deleted file mode 100644 index 8d3b0ed..0000000 --- a/src/json2sql.egg-info/requires.txt +++ /dev/null @@ -1,9 +0,0 @@ -typer>=0.9.0 -rich>=13.0.0 - -[dev] -pytest>=7.0 -pytest-cov - -[license] -revenueholdings-license>=0.1.0 diff --git a/src/json2sql.egg-info/top_level.txt b/src/json2sql.egg-info/top_level.txt deleted file mode 100644 index 5ad8bab..0000000 --- a/src/json2sql.egg-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -json2sql diff --git a/src/json2sql/converter.py b/src/json2sql/converter.py index a326f8c..e19916b 100644 --- a/src/json2sql/converter.py +++ b/src/json2sql/converter.py @@ -93,7 +93,10 @@ def _convert_objects(self, objects: list[dict], table_name: str) -> str: # When flattening, compute the full column set first so rows align if self.flatten: columns, flat_map = self._infer_columns_flattened(objects, table_name) - # Process nested arrays into child tables + # Process nested arrays into child tables, grouped by key so that + # each nested array produces exactly ONE child table whose INSERT + # covers every parent row's children. + nested_groups: dict[str, tuple[list[dict], list[dict]]] = {} for obj in objects: for key, value in obj.items(): if ( @@ -101,7 +104,11 @@ def _convert_objects(self, objects: list[dict], table_name: str) -> str: and value and all(isinstance(v, dict) for v in value) ): - self._flatten_nested(table_name, key, value, obj) + children, parents = nested_groups.setdefault(key, ([], [])) + children.extend(value) + parents.extend([obj] * len(value)) + for key, (children, parents) in nested_groups.items(): + self._flatten_nested(table_name, key, children, parents) else: columns = self._infer_columns(objects) flat_map = {} @@ -240,27 +247,50 @@ def _flatten_nested( parent_table: str, key: str, nested_objects: list[dict], - parent_obj: dict, + parent_objs: list[dict], ) -> None: - """Flatten a nested array of objects into a separate table.""" + """Flatten nested arrays of objects into a single child table. + + ``nested_objects`` and ``parent_objs`` are aligned lists: each child + row links back to its own parent via the foreign key. Grouping all + parents' children into one table avoids emitting duplicate + ``CREATE TABLE`` statements when multiple rows carry nested arrays. + """ child_table = f"{parent_table}_{key}" columns = self._infer_columns(nested_objects) - # Add parent reference — only if no existing column has the FK name + # Add parent reference — only if no existing column has the FK name. + # Prefer explicit ID fields over generic "name" to ensure the FK column + # type matches the parent table's primary key type. parent_ref = None - for pk in ("id", "name", parent_table + "_id"): - if pk in parent_obj: + # Priority order for parent reference key: + # 1. "id" (generic primary key) + # 2. "{parent_table}_id" (table-specific, e.g., "users_id") + # 3. Any key ending in "_id" found in parent objects (e.g., "user_id") + # 4. "name" (fallback only when no ID-like field exists) + candidate_keys = ["id", f"{parent_table}_id"] + # Add any *_id keys found in parent objects (excluding already listed) + seen = set(candidate_keys) + for obj in parent_objs: + for k in obj: + if k.endswith("_id") and k not in seen: + candidate_keys.append(k) + seen.add(k) + candidate_keys.append("name") + for pk in candidate_keys: + if any(pk in parent_obj for parent_obj in parent_objs): parent_ref = pk break fk_col = f"{parent_table}_{parent_ref}" if parent_ref else None fk_already_exists = fk_col and fk_col in columns if fk_col and not fk_already_exists: + fk_parent = next(p for p in parent_objs if parent_ref in p) columns = { - fk_col: sql_type_for(parent_obj[parent_ref], self.dialect), + fk_col: sql_type_for(fk_parent[parent_ref], self.dialect), **columns, } rows: list[list[str]] = [] - for nested in nested_objects: + for nested, parent_obj in zip(nested_objects, parent_objs, strict=True): row: list[str] = [] for col_name in columns: if col_name == fk_col and not fk_already_exists: @@ -277,6 +307,7 @@ def _process_flatten(self, objects: list, table_name: str) -> None: return if not objects or not isinstance(objects[0], dict): return + nested_groups: dict[str, tuple[list[dict], list[dict]]] = {} for obj in objects: for key, value in obj.items(): if ( @@ -284,4 +315,8 @@ def _process_flatten(self, objects: list, table_name: str) -> None: and value and all(isinstance(v, dict) for v in value) ): - self._flatten_nested(table_name, key, value, obj) + children, parents = nested_groups.setdefault(key, ([], [])) + children.extend(value) + parents.extend([obj] * len(value)) + for key, (children, parents) in nested_groups.items(): + self._flatten_nested(table_name, key, children, parents) diff --git a/tests/test_edge_cases.py b/tests/test_edge_cases.py index 5ac5b73..fa32f7e 100644 --- a/tests/test_edge_cases.py +++ b/tests/test_edge_cases.py @@ -108,3 +108,41 @@ def test_convert_objects_list_vs_dict_root(self): result = converter.convert(json.dumps([{"name": "test"}])) assert "INSERT INTO" in result assert "'test'" in result + + +def test_flatten_multiple_parent_rows_single_child_table(): + """Multiple parent rows with nested arrays yield ONE child table with all rows.""" + import json as _json + + from json2sql.converter import JSONToSQLConverter + + data = [ + {"id": 1, "name": "a", "tags": [{"label": "x", "score": 1}]}, + { + "id": 2, + "name": "b", + "tags": [{"label": "y", "score": 2}, {"label": "z", "score": 3}], + }, + ] + text = _json.dumps(data) + out = JSONToSQLConverter(flatten=True).convert(text, "users") + assert out.count('CREATE TABLE "users_tags"') == 1 + assert "'z', 3" in out and "'y', 2" in out and "'x', 1" in out + + schema = JSONToSQLConverter(flatten=True).generate_schema(text, "users") + assert schema.count('CREATE TABLE "users_tags"') == 1 + + +def test_flatten_child_rows_keep_own_parent_fk(): + """Each child row links to its own parent via the FK column.""" + import json as _json + + from json2sql.converter import JSONToSQLConverter + + data = [ + {"id": 10, "items": [{"sku": "a1"}]}, + {"id": 20, "items": [{"sku": "b1"}]}, + ] + out = JSONToSQLConverter(flatten=True).convert(_json.dumps(data), "orders") + assert "(10, 'a1')" in out + assert "(20, 'b1')" in out diff --git a/tests/test_type_inference.py b/tests/test_type_inference.py index cf6d693..b4e7a98 100644 --- a/tests/test_type_inference.py +++ b/tests/test_type_inference.py @@ -131,3 +131,72 @@ def test_convert_never_emits_empty_column_list(): ) assert "();" not in out assert "INSERT INTO" in out + + +class TestFlattenFKDetection: + """Tests for correct FK column detection in flatten mode. + + The FK column in a child table must match the parent table's primary key + column name and type. Previously the code preferred "name" over explicit + ID fields like "user_id" or "users_id", causing a type mismatch. + """ + + @pytest.mark.parametrize("dialect", [Dialect.POSTGRES, Dialect.MYSQL, Dialect.SQLITE]) + def test_flatten_prefers_id_over_name(self, dialect): + """When parent has both 'id' and 'name', 'id' should be used for FK.""" + conv = JSONToSQLConverter(dialect=dialect, flatten=True) + data = json.dumps([ + {"id": 1, "name": "Alice", "tags": [{"label": "x"}]}, + {"id": 2, "name": "Bob", "tags": [{"label": "y"}]}, + ]) + out = conv.convert(data, table_name="users") + # FK column should be users_id (from parent's id), not users_name + assert '"users_id"' in out or '`users_id`' in out + assert '"users_name"' not in out and '`users_name`' not in out + # Parent table should have id column + assert '"id"' in out or '`id`' in out + + @pytest.mark.parametrize("dialect", [Dialect.POSTGRES, Dialect.MYSQL, Dialect.SQLITE]) + def test_flatten_prefers_table_specific_id(self, dialect): + """When parent has '{table}_id' (e.g., users_id), it should be used.""" + conv = JSONToSQLConverter(dialect=dialect, flatten=True) + data = json.dumps([ + {"users_id": 10, "name": "Alice", "tags": [{"label": "x"}]}, + {"users_id": 20, "name": "Bob", "tags": [{"label": "y"}]}, + ]) + out = conv.convert(data, table_name="users") + # FK column should be users_users_id (from parent's users_id) + assert '"users_users_id"' in out or '`users_users_id`' in out + assert '"users_name"' not in out and '`users_name`' not in out + + @pytest.mark.parametrize("dialect", [Dialect.POSTGRES, Dialect.MYSQL, Dialect.SQLITE]) + def test_flatten_prefers_any_id_suffix(self, dialect): + """When parent has a singular '*_id' (e.g., user_id), it should be used over 'name'.""" + conv = JSONToSQLConverter(dialect=dialect, flatten=True) + data = json.dumps([ + {"user_id": 100, "name": "Alice", "tags": [{"label": "x"}]}, + {"user_id": 200, "name": "Bob", "tags": [{"label": "y"}]}, + ]) + out = conv.convert(data, table_name="users") + # FK column should be users_user_id (from parent's user_id) + assert '"users_user_id"' in out or '`users_user_id`' in out + assert '"users_name"' not in out and '`users_name`' not in out + # FK type should be numeric (matching parent's user_id type) + if dialect == Dialect.MYSQL: + assert "INT" in out + else: + assert "INTEGER" in out + + @pytest.mark.parametrize("dialect", [Dialect.POSTGRES, Dialect.MYSQL, Dialect.SQLITE]) + def test_flatten_fallback_to_name_when_no_id(self, dialect): + """When parent has no ID-like field, 'name' is used as fallback.""" + conv = JSONToSQLConverter(dialect=dialect, flatten=True) + data = json.dumps([ + {"name": "Alice", "tags": [{"label": "x"}]}, + {"name": "Bob", "tags": [{"label": "y"}]}, + ]) + out = conv.convert(data, table_name="users") + # FK column should be users_name (fallback) + assert '"users_name"' in out or '`users_name`' in out + # Parent table should have name column + assert '"name"' in out or '`name`' in out