Support JSON lines with missing columns (#7170)

* Test cast_table_to_schema with missing fields

* Support cast_table_to_schema with missing fields

src/datasets/table.py

@@ -2235,13 +2235,20 @@ def cast_table_to_schema(table: pa.Table, schema: pa.Schema):
     from .features import Features
 
     features = Features.from_arrow_schema(schema)
-    if sorted(table.column_names) != sorted(features):
+    table_column_names = set(table.column_names)
+    if not table_column_names <= set(schema.names):
         raise CastError(
             f"Couldn't cast\n{_short_str(table.schema)}\nto\n{_short_str(features)}\nbecause column names don't match",
             table_column_names=table.column_names,
             requested_column_names=list(features),
         )
-    arrays = [cast_array_to_feature(table[name], feature) for name, feature in features.items()]
+    arrays = [
+        cast_array_to_feature(
+            table[name] if name in table_column_names else pa.array([None] * len(table), type=schema.field(name).type),
+            feature,
+        )
+        for name, feature in features.items()
+    ]
     return pa.Table.from_arrays(arrays, schema=schema)
 
 

tests/test_table.py

@@ -23,6 +23,7 @@
     _memory_mapped_arrow_table_from_file,
     array_cast,
     cast_array_to_feature,
+    cast_table_to_schema,
     concat_tables,
     embed_array_storage,
     embed_table_storage,
@@ -1442,3 +1443,11 @@ def test_array_cast(from_type, to_type):
     cast_arr = array_cast(arr, array_type[to_type])
     assert cast_arr.type == array_type[to_type]
     assert cast_arr.values == arr.values
+
+
+def test_cast_table_to_schema_with_missing_fields():
+    table = pa.table({"age": [25, 63]})
+    schema = pa.schema({"age": pa.int32(), "name": pa.string()})
+    cast_table = cast_table_to_schema(table, schema)
+    assert cast_table.schema == pa.schema({"age": pa.int32(), "name": pa.string()})
+    assert cast_table.to_pydict() == {"age": [25, 63], "name": [None, None]}