diff --git a/docs/api/collection/metadata.rst b/docs/api/collection/metadata.rst index 907ff2c2..ec5de538 100644 --- a/docs/api/collection/metadata.rst +++ b/docs/api/collection/metadata.rst @@ -10,6 +10,8 @@ Metadata Collection.member_schemas Collection.required_members Collection.optional_members + Collection.lazy_members + Collection.eager_members Collection.non_ignored_members Collection.ignored_members Collection.common_primary_key diff --git a/docs/api/columns/index.rst b/docs/api/columns/index.rst index 634cef7c..67ea36a5 100644 --- a/docs/api/columns/index.rst +++ b/docs/api/columns/index.rst @@ -12,6 +12,7 @@ Columns Binary Bool Categorical + Column Date Datetime Decimal diff --git a/docs/api/filter_result/failure_info.rst b/docs/api/filter_result/failure_info.rst index d0301925..054d1f4c 100644 --- a/docs/api/filter_result/failure_info.rst +++ b/docs/api/filter_result/failure_info.rst @@ -18,6 +18,7 @@ Inspection :toctree: _gen/ FailureInfo.invalid + FailureInfo.details FailureInfo.counts FailureInfo.cooccurrence_counts FailureInfo.__len__ diff --git a/docs/api/misc/index.rst b/docs/api/misc/index.rst index a6a027df..100407b8 100644 --- a/docs/api/misc/index.rst +++ b/docs/api/misc/index.rst @@ -7,6 +7,7 @@ Miscellaneous :toctree: _gen/ :nosignatures: + CollectionMember Config random.Generator testing.create_schema diff --git a/docs/api/schema/conversion.rst b/docs/api/schema/conversion.rst index 379efc95..106ed1ab 100644 --- a/docs/api/schema/conversion.rst +++ b/docs/api/schema/conversion.rst @@ -7,3 +7,4 @@ Conversion :toctree: _gen/ Schema.to_sqlalchemy_columns + Schema.to_pydantic_model diff --git a/docs/conf.py b/docs/conf.py index 6e49c8b4..8ad1627c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -95,7 +95,6 @@ ## myst_parser myst_parser_config = {"myst_enable_extensions": ["rst_eval_roles"]} -nitpick_ignore = [("myst", "group-rules")] ## numpydoc numpydoc_class_members_toctree = False diff --git a/docs/guides/examples/real-world.ipynb b/docs/guides/examples/real-world.ipynb index f47e3788..1e33ee1d 100644 --- a/docs/guides/examples/real-world.ipynb +++ b/docs/guides/examples/real-world.ipynb @@ -12,7 +12,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -81,7 +81,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -126,19 +126,19 @@ " white-space: pre-wrap;\n", "}\n", "\n", - "shape: (3, 5)
invoice_idadmission_datedischarge_datereceived_atamount
strdatedatedatetime[μs]decimal[*,0]
"001"2025-01-012025-01-042025-01-05 00:00:001000
"002"2025-01-052025-01-072025-01-08 00:00:00200
"003"2025-01-012025-01-012025-01-02 00:00:00400
" + "shape: (3, 5)
invoice_idadmission_datedischarge_datereceived_atamount
strdatedatedatetime[μs]decimal[38,0]
"001"2025-01-012025-01-042025-01-05 00:00:001000
"002"2025-01-052025-01-072025-01-08 00:00:00200
"003"2025-01-012025-01-012025-01-02 00:00:00400
" ], "text/plain": [ "shape: (3, 5)\n", - "┌────────────┬────────────────┬────────────────┬─────────────────────┬──────────────┐\n", - "│ invoice_id ┆ admission_date ┆ discharge_date ┆ received_at ┆ amount │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ str ┆ date ┆ date ┆ datetime[μs] ┆ decimal[*,0] │\n", - "╞════════════╪════════════════╪════════════════╪═════════════════════╪══════════════╡\n", - "│ 001 ┆ 2025-01-01 ┆ 2025-01-04 ┆ 2025-01-05 00:00:00 ┆ 1000 │\n", - "│ 002 ┆ 2025-01-05 ┆ 2025-01-07 ┆ 2025-01-08 00:00:00 ┆ 200 │\n", - "│ 003 ┆ 2025-01-01 ┆ 2025-01-01 ┆ 2025-01-02 00:00:00 ┆ 400 │\n", - "└────────────┴────────────────┴────────────────┴─────────────────────┴──────────────┘" + "┌────────────┬────────────────┬────────────────┬─────────────────────┬───────────────┐\n", + "│ invoice_id ┆ admission_date ┆ discharge_date ┆ received_at ┆ amount │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ str ┆ date ┆ date ┆ datetime[μs] ┆ decimal[38,0] │\n", + "╞════════════╪════════════════╪════════════════╪═════════════════════╪═══════════════╡\n", + "│ 001 ┆ 2025-01-01 ┆ 2025-01-04 ┆ 2025-01-05 00:00:00 ┆ 1000 │\n", + "│ 002 ┆ 2025-01-05 ┆ 2025-01-07 ┆ 2025-01-08 00:00:00 ┆ 200 │\n", + "│ 003 ┆ 2025-01-01 ┆ 2025-01-01 ┆ 2025-01-02 00:00:00 ┆ 400 │\n", + "└────────────┴────────────────┴────────────────┴─────────────────────┴───────────────┘" ] }, "execution_count": 5, @@ -168,7 +168,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "If the data to validate contains invalid rows, `dataframely` will raise a `RuleValidationError` with a summary about the violated validation rules." + "If the data to validate contains invalid rows, `dataframely` will raise a `ValidationError` with a summary about the violated validation rules." ] }, { @@ -177,15 +177,15 @@ "metadata": {}, "outputs": [ { - "ename": "RuleValidationError", + "ename": "ValidationError", "evalue": "1 rules failed validation:\n * Column 'amount' failed validation for 1 rules:\n - 'min_exclusive' failed for 1 rows", "output_type": "error", "traceback": [ "\u001b[31m---------------------------------------------------------------------------\u001b[39m", - "\u001b[31mRuleValidationError\u001b[39m Traceback (most recent call last)", - "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[6]\u001b[39m\u001b[32m, line 10\u001b[39m\n\u001b[32m 1\u001b[39m \u001b[38;5;66;03m# Raise during validation if there are invalid rows\u001b[39;00m\n\u001b[32m 2\u001b[39m invoices = pl.DataFrame({\n\u001b[32m 3\u001b[39m \u001b[33m\"\u001b[39m\u001b[33minvoice_id\u001b[39m\u001b[33m\"\u001b[39m: [\u001b[33m\"\u001b[39m\u001b[33m001\u001b[39m\u001b[33m\"\u001b[39m, \u001b[33m\"\u001b[39m\u001b[33m002\u001b[39m\u001b[33m\"\u001b[39m, \u001b[33m\"\u001b[39m\u001b[33m003\u001b[39m\u001b[33m\"\u001b[39m],\n\u001b[32m 4\u001b[39m \u001b[33m\"\u001b[39m\u001b[33madmission_date\u001b[39m\u001b[33m\"\u001b[39m: [date(\u001b[32m2025\u001b[39m, \u001b[32m1\u001b[39m, \u001b[32m1\u001b[39m), date(\u001b[32m2025\u001b[39m, \u001b[32m1\u001b[39m, \u001b[32m5\u001b[39m), date(\u001b[32m2025\u001b[39m, \u001b[32m1\u001b[39m, \u001b[32m1\u001b[39m)],\n\u001b[32m (...)\u001b[39m\u001b[32m 7\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mamount\u001b[39m\u001b[33m\"\u001b[39m: [\u001b[32m0.0\u001b[39m, \u001b[32m200.0\u001b[39m, \u001b[32m400.0\u001b[39m], \u001b[38;5;66;03m# Invalid amount `0.0` here\u001b[39;00m\n\u001b[32m 8\u001b[39m })\n\u001b[32m---> \u001b[39m\u001b[32m10\u001b[39m \u001b[43mInvoiceSchema\u001b[49m\u001b[43m.\u001b[49m\u001b[43mvalidate\u001b[49m\u001b[43m(\u001b[49m\u001b[43minvoices\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcast\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/workspace/dataframely/dataframely/schema.py:280\u001b[39m, in \u001b[36mSchema.validate\u001b[39m\u001b[34m(cls, df, cast)\u001b[39m\n\u001b[32m 278\u001b[39m df_valid, failures = \u001b[38;5;28mcls\u001b[39m.filter(df, cast=cast)\n\u001b[32m 279\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(failures) > \u001b[32m0\u001b[39m:\n\u001b[32m--> \u001b[39m\u001b[32m280\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m RuleValidationError(failures.counts())\n\u001b[32m 281\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m df_valid\n", - "\u001b[31mRuleValidationError\u001b[39m: 1 rules failed validation:\n * Column 'amount' failed validation for 1 rules:\n - 'min_exclusive' failed for 1 rows" + "\u001b[31mValidationError\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[6]\u001b[39m\u001b[32m, line 16\u001b[39m\n\u001b[32m 12\u001b[39m \u001b[33m\"amount\"\u001b[39m: [\u001b[32m0.0\u001b[39m, \u001b[32m200.0\u001b[39m, \u001b[32m400.0\u001b[39m], \u001b[38;5;66;03m# Invalid amount `0.0` here\u001b[39;00m\n\u001b[32m 13\u001b[39m }\n\u001b[32m 14\u001b[39m )\n\u001b[32m 15\u001b[39m \n\u001b[32m---> \u001b[39m\u001b[32m16\u001b[39m InvoiceSchema.validate(invoices, cast=\u001b[38;5;28;01mTrue\u001b[39;00m)\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/Repos/dataframely/dataframely/schema.py:543\u001b[39m, in \u001b[36mSchema.validate\u001b[39m\u001b[34m(cls, df, cast, **kwargs)\u001b[39m\n\u001b[32m 541\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(failure) > \u001b[32m0\u001b[39m:\n\u001b[32m 542\u001b[39m counts = failure.counts()\n\u001b[32m--> \u001b[39m\u001b[32m543\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m ValidationError(\n\u001b[32m 544\u001b[39m format_rule_failures(\n\u001b[32m 545\u001b[39m \u001b[38;5;28mlist\u001b[39m(counts.items()),\n\u001b[32m 546\u001b[39m failures_from=failure._df.select(counts.keys()),\n\u001b[32m 547\u001b[39m examples_from=failure.invalid(),\n\u001b[32m 548\u001b[39m primary_key_columns=\u001b[38;5;28mcls\u001b[39m.primary_key(),\n\u001b[32m 549\u001b[39m max_examples=Config.options[\u001b[33m\"\u001b[39m\u001b[33mmax_failure_examples\u001b[39m\u001b[33m\"\u001b[39m],\n\u001b[32m 550\u001b[39m )\n\u001b[32m 551\u001b[39m )\n\u001b[32m 552\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m out\n\u001b[32m 553\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n", + "\u001b[31mValidationError\u001b[39m: 1 rules failed validation:\n * Column 'amount' failed validation for 1 rules:\n - 'min_exclusive' failed for 1 rows" ] } ], @@ -284,17 +284,17 @@ " white-space: pre-wrap;\n", "}\n", "\n", - "shape: (1, 5)
invoice_idadmission_datedischarge_datereceived_atamount
strdatedatedatetime[μs]decimal[*,0]
"001"2025-01-012025-01-042025-01-05 00:00:000
" + "shape: (1, 5)
invoice_idadmission_datedischarge_datereceived_atamount
strdatedatedatetime[μs]f64
"001"2025-01-012025-01-042025-01-05 00:00:000.0
" ], "text/plain": [ "shape: (1, 5)\n", - "┌────────────┬────────────────┬────────────────┬─────────────────────┬──────────────┐\n", - "│ invoice_id ┆ admission_date ┆ discharge_date ┆ received_at ┆ amount │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ str ┆ date ┆ date ┆ datetime[μs] ┆ decimal[*,0] │\n", - "╞════════════╪════════════════╪════════════════╪═════════════════════╪══════════════╡\n", - "│ 001 ┆ 2025-01-01 ┆ 2025-01-04 ┆ 2025-01-05 00:00:00 ┆ 0 │\n", - "└────────────┴────────────────┴────────────────┴─────────────────────┴──────────────┘" + "┌────────────┬────────────────┬────────────────┬─────────────────────┬────────┐\n", + "│ invoice_id ┆ admission_date ┆ discharge_date ┆ received_at ┆ amount │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ str ┆ date ┆ date ┆ datetime[μs] ┆ f64 │\n", + "╞════════════╪════════════════╪════════════════╪═════════════════════╪════════╡\n", + "│ 001 ┆ 2025-01-01 ┆ 2025-01-04 ┆ 2025-01-05 00:00:00 ┆ 0.0 │\n", + "└────────────┴────────────────┴────────────────┴─────────────────────┴────────┘" ] }, "execution_count": 10, @@ -320,7 +320,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ @@ -346,7 +346,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -453,15 +453,15 @@ "metadata": {}, "outputs": [ { - "ename": "MemberValidationError", - "evalue": "2 members failed validation:\n > Member 'invoices' failed validation:\n 1 rules failed validation:\n - 'at_least_one_diagnosis_per_invoice' failed validation for 1 rows\n > Member 'diagnoses' failed validation:\n 0 rules failed validation:", + "ename": "ValidationError", + "evalue": "1 members failed validation:\n > Member 'invoices' failed validation:\n 1 rules failed validation:\n - 'at_least_one_diagnosis_per_invoice' failed for 1 rows", "output_type": "error", "traceback": [ "\u001b[31m---------------------------------------------------------------------------\u001b[39m", - "\u001b[31mMemberValidationError\u001b[39m Traceback (most recent call last)", - "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[15]\u001b[39m\u001b[32m, line 23\u001b[39m\n\u001b[32m 1\u001b[39m invoices = pl.DataFrame(\n\u001b[32m 2\u001b[39m {\n\u001b[32m 3\u001b[39m \u001b[33m\"\u001b[39m\u001b[33minvoice_id\u001b[39m\u001b[33m\"\u001b[39m: [\u001b[33m\"\u001b[39m\u001b[33m001\u001b[39m\u001b[33m\"\u001b[39m, \u001b[33m\"\u001b[39m\u001b[33m002\u001b[39m\u001b[33m\"\u001b[39m, \u001b[33m\"\u001b[39m\u001b[33m003\u001b[39m\u001b[33m\"\u001b[39m],\n\u001b[32m (...)\u001b[39m\u001b[32m 12\u001b[39m }\n\u001b[32m 13\u001b[39m )\n\u001b[32m 15\u001b[39m diagnoses = pl.DataFrame(\n\u001b[32m 16\u001b[39m {\n\u001b[32m 17\u001b[39m \u001b[33m\"\u001b[39m\u001b[33minvoice_id\u001b[39m\u001b[33m\"\u001b[39m: [\u001b[33m\"\u001b[39m\u001b[33m001\u001b[39m\u001b[33m\"\u001b[39m, \u001b[33m\"\u001b[39m\u001b[33m001\u001b[39m\u001b[33m\"\u001b[39m, \u001b[33m\"\u001b[39m\u001b[33m002\u001b[39m\u001b[33m\"\u001b[39m],\n\u001b[32m (...)\u001b[39m\u001b[32m 20\u001b[39m }\n\u001b[32m 21\u001b[39m )\n\u001b[32m---> \u001b[39m\u001b[32m23\u001b[39m claims = \u001b[43mHospitalClaims\u001b[49m\u001b[43m.\u001b[49m\u001b[43mvalidate\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 24\u001b[39m \u001b[43m \u001b[49m\u001b[43m{\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43minvoices\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43minvoices\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mdiagnoses\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mdiagnoses\u001b[49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcast\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\n\u001b[32m 25\u001b[39m \u001b[43m)\u001b[49m\n\u001b[32m 27\u001b[39m \u001b[38;5;66;03m# Aggregate diagnoses per invoice\u001b[39;00m\n\u001b[32m 28\u001b[39m \u001b[38;5;28mprint\u001b[39m(\n\u001b[32m 29\u001b[39m claims.invoices.join(claims.diagnoses, on=\u001b[33m\"\u001b[39m\u001b[33minvoice_id\u001b[39m\u001b[33m\"\u001b[39m, how=\u001b[33m\"\u001b[39m\u001b[33minner\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m 30\u001b[39m .group_by(\u001b[33m\"\u001b[39m\u001b[33minvoice_id\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m (...)\u001b[39m\u001b[32m 38\u001b[39m .collect()\n\u001b[32m 39\u001b[39m )\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/workspace/dataframely/dataframely/collection.py:82\u001b[39m, in \u001b[36mCollection.validate\u001b[39m\u001b[34m(cls, data, cast)\u001b[39m\n\u001b[32m 80\u001b[39m out, failure = \u001b[38;5;28mcls\u001b[39m.filter(data, cast=cast)\n\u001b[32m 81\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28many\u001b[39m(\u001b[38;5;28mlen\u001b[39m(fail) > \u001b[32m0\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m fail \u001b[38;5;129;01min\u001b[39;00m failure.values()):\n\u001b[32m---> \u001b[39m\u001b[32m82\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m MemberValidationError(\n\u001b[32m 83\u001b[39m {\n\u001b[32m 84\u001b[39m name: RuleValidationError(fail.counts())\n\u001b[32m 85\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m name, fail \u001b[38;5;129;01min\u001b[39;00m failure.items()\n\u001b[32m 86\u001b[39m }\n\u001b[32m 87\u001b[39m )\n\u001b[32m 88\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m out\n", - "\u001b[31mMemberValidationError\u001b[39m: 2 members failed validation:\n > Member 'invoices' failed validation:\n 1 rules failed validation:\n - 'at_least_one_diagnosis_per_invoice' failed validation for 1 rows\n > Member 'diagnoses' failed validation:\n 0 rules failed validation:" + "\u001b[31mValidationError\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[15]\u001b[39m\u001b[32m, line 23\u001b[39m\n\u001b[32m 19\u001b[39m \u001b[33m\"is_main\"\u001b[39m: [\u001b[38;5;28;01mTrue\u001b[39;00m, \u001b[38;5;28;01mFalse\u001b[39;00m, \u001b[38;5;28;01mTrue\u001b[39;00m],\n\u001b[32m 20\u001b[39m }\n\u001b[32m 21\u001b[39m )\n\u001b[32m 22\u001b[39m \n\u001b[32m---> \u001b[39m\u001b[32m23\u001b[39m claims = HospitalClaims.validate(\n\u001b[32m 24\u001b[39m {\u001b[33m\"invoices\"\u001b[39m: invoices, \u001b[33m\"diagnoses\"\u001b[39m: diagnoses}, cast=\u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[32m 25\u001b[39m )\n\u001b[32m 26\u001b[39m \n", + "\u001b[36mFile \u001b[39m\u001b[32m~/Repos/dataframely/dataframely/collection/collection.py:430\u001b[39m, in \u001b[36mCollection.validate\u001b[39m\u001b[34m(cls, data, cast, lazy, skip_member_validation, **kwargs)\u001b[39m\n\u001b[32m 422\u001b[39m details = [\n\u001b[32m 423\u001b[39m \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33m > Member \u001b[39m\u001b[33m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmember\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m'\u001b[39m\u001b[33m failed validation:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[33m\"\u001b[39m\n\u001b[32m 424\u001b[39m + textwrap.indent(error, \u001b[33m\"\u001b[39m\u001b[33m \u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m 425\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m member, error \u001b[38;5;129;01min\u001b[39;00m errors.items()\n\u001b[32m 426\u001b[39m ]\n\u001b[32m 427\u001b[39m message = \u001b[33m\"\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[33m\"\u001b[39m.join(\n\u001b[32m 428\u001b[39m [\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mlen\u001b[39m(errors)\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m members failed validation:\u001b[39m\u001b[33m\"\u001b[39m] + details\n\u001b[32m 429\u001b[39m )\n\u001b[32m--> \u001b[39m\u001b[32m430\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m ValidationError(message)\n\u001b[32m 431\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m filtered\n\u001b[32m 432\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m 433\u001b[39m \u001b[38;5;66;03m# If we do NOT perform the validation eagerly, we can perform it more\u001b[39;00m\n\u001b[32m 434\u001b[39m \u001b[38;5;66;03m# efficiently as we cannot easily propagate error messages from different\u001b[39;00m\n\u001b[32m 435\u001b[39m \u001b[38;5;66;03m# members anyways.\u001b[39;00m\n", + "\u001b[31mValidationError\u001b[39m: 1 members failed validation:\n > Member 'invoices' failed validation:\n 1 rules failed validation:\n - 'at_least_one_diagnosis_per_invoice' failed for 1 rows" ] } ], diff --git a/docs/guides/faq.md b/docs/guides/faq.md index c9fdc463..3bb76408 100644 --- a/docs/guides/faq.md +++ b/docs/guides/faq.md @@ -5,25 +5,19 @@ thinking, please add it here. ## How do I define additional unique keys in a {class}`~dataframely.Schema`? -By default, `dataframely` only supports defining a single non-nullable (composite) primary key in {class} -`~dataframely.Schema`. -However, in some scenarios it may be useful to define additional unique keys (which support nullable fields and/or which -are additionally unique). +`dataframely` only supports defining a single non-nullable (composite) primary key in {class}`~dataframely.Schema`. +However, in some scenarios it may be useful to define additional unique keys. -Consider the following example, which demonstrates two rules: one for validating that a field is entirely unique, and -another for validating that a field, when provided, is unique. +For individual columns, set `unique=True` to require that the column's values are unique. Note that multiple `null` +values are considered duplicates. To require uniqueness only for non-null values, or uniqueness across a combination of +columns, use a custom rule instead: ```python class UserSchema(dy.Schema): - user_id = dy.UInt64(primary_key=True, nullable=False) - username = dy.String(nullable=False) + user_id = dy.UInt64(primary_key=True) + username = dy.String(unique=True) email = dy.String(nullable=True) # Must be unique, or null. - @dy.rule() - def unique_username(cls) -> pl.Expr: - """Username, a non-nullable field, must be total unique.""" - return (pl.len() == 1).over("username") - @dy.rule() def unique_email_or_null(cls) -> pl.Expr: """Email must be unique, if provided.""" @@ -32,7 +26,7 @@ class UserSchema(dy.Schema): ## How do I fix the ruff error `First argument of a method should be named self`? -See our documentation on [group rules](./quickstart.md#group-rules). +See our documentation on [custom rules](./quickstart.md#custom-rules). ## How do I get a {class}`~dataframely.Schema` for my dataframe? diff --git a/docs/guides/features/column-metadata.md b/docs/guides/features/column-metadata.md index b850f830..ae33a78c 100644 --- a/docs/guides/features/column-metadata.md +++ b/docs/guides/features/column-metadata.md @@ -26,6 +26,13 @@ class UserSchema(dy.Schema): Metadata are never read by `dataframely` and merely enable users to provide custom information in a structured way. +For human-readable column descriptions, all column types additionally provide a dedicated `description` parameter: + +```python +class UserSchema(dy.Schema): + id = dy.String(primary_key=True, description="The unique identifier of the user.") +``` + ```{note} User-provided metadata can be useful for code generation. For instance, one could specify metadata for columns, such as database column types or constraints, and override the built-in SQLAlchemy generation for more tailored SQL output. ``` diff --git a/docs/guides/features/data-generation.md b/docs/guides/features/data-generation.md index 49a67f5c..93419971 100644 --- a/docs/guides/features/data-generation.md +++ b/docs/guides/features/data-generation.md @@ -54,7 +54,7 @@ class InvoiceSchema(dy.Schema): @dy.rule() def discharge_after_admission(cls) -> pl.Expr: - return InvoiceSchema.discharge_date.col >= InvoiceSchema.admission_date.col + return cls.discharge_date.col >= cls.admission_date.col # `@dy.rule`s will be respected as well for data generation. @@ -86,8 +86,8 @@ df: dy.DataFrame[InvoiceSchema] = InvoiceSchema.sample( overrides={ # Use either ..name or just the column name as a string. InvoiceSchema.invoice_id.name: ["1234567890", "2345678901", "3456789012"], - # Dataframely will automatically infer the number of rows based on the longest given - # sequence of values and broadcast all other columns to that shape. + # Dataframely will automatically infer the number of rows based on the length of the + # given sequences of values and broadcast scalar values to that shape. "admission_date": date(2025, 1, 1), } ) @@ -110,7 +110,7 @@ df: dy.DataFrame[InvoiceSchema] = InvoiceSchema.sample( ### Providing custom column overrides -Complex validation rules (such as dependencies between columns or ordering criteria) can cause sampling to run excessively long or fail. Specifying additional `overrides` individually in such cases can be tedious. Instead, the {meth}`~dataframely.Schema._sampling_overrides` hook on a schema can be used to specify polars expressions for columns in the schema that will be applied before generated data is filtered during sampling. +Complex validation rules (such as dependencies between columns or ordering criteria) can cause sampling to run excessively long or fail. Specifying additional `overrides` individually in such cases can be tedious. Instead, the `Schema._sampling_overrides` hook on a schema can be used to specify polars expressions for columns in the schema that will be applied before generated data is filtered during sampling. ```python import polars as pl @@ -174,16 +174,16 @@ class HospitalInvoiceData(dy.Collection): diagnosis: dy.LazyFrame[DiagnosisSchema] @dy.filter() - def at_least_one_diagnosis(cls) -> pl.Expr: + def at_least_one_diagnosis(self) -> pl.LazyFrame: return dy.functional.require_relationship_one_to_at_least_one( - cls.invoice, - cls.diagnosis, + self.invoice, + self.diagnosis, on="invoice_id", ) ``` In this case, calling {meth}`~dataframely.Collection.sample` will fail, because dataframely does not parse the body of `at_least_one_diagnosis` which may contain arbitrary polars expressions. -To address the problem, one can override {meth}`~dataframely.Collection._preprocess_sample` to generate a random number of diagnoses per invoice: +To address the problem, one can override `Collection._preprocess_sample` to generate a random number of diagnoses per invoice: ```python from random import random @@ -197,10 +197,10 @@ class HospitalInvoiceData(dy.Collection): diagnosis: dy.LazyFrame[DiagnosisSchema] @dy.filter() - def at_least_one_diagnosis(cls) -> pl.Expr: + def at_least_one_diagnosis(self) -> pl.LazyFrame: return dy.functional.require_relationship_one_to_at_least_one( - cls.invoice, - cls.diagnosis, + self.invoice, + self.diagnosis, on="invoice_id", ) @@ -240,7 +240,7 @@ class OutputSchema(dy.Schema): # function under test def get_diabetes_invoice_amounts( - invoice_data: HospitalClaims, + invoice_data: HospitalInvoiceData, ) -> dy.LazyFrame[OutputSchema]: return OutputSchema.cast( invoice_data.diagnosis.filter(DiagnosisSchema.code.col.str.starts_with("E11")) diff --git a/docs/guides/features/lazy-validation.md b/docs/guides/features/lazy-validation.md index 61e31beb..92b0d625 100644 --- a/docs/guides/features/lazy-validation.md +++ b/docs/guides/features/lazy-validation.md @@ -3,39 +3,47 @@ In many cases, dataframely's capability to validate and filter input data is used at core application boundaries. As a result, `validate` and `filter` are generally expected to be used at points where `collect` is called on a lazy frame. However, there may be situations where validation or filtering should simply be added to the lazy computation -graph. Starting in dataframely v2, this is supported via a custom polars plugin. +graph. This is supported via a custom polars plugin. -## The `eager` parameter +## Schemas: lazy-in-lazy-out -All of the following methods expose an `eager: bool` parameter: +{meth}`Schema.validate() ` and {meth}`Schema.filter() ` follow +the same "eager-in-eager-out" and "lazy-in-lazy-out" semantics as polars: if a {class}`~polars.DataFrame` is passed, +validation and filtering are performed immediately. If a {class}`~polars.LazyFrame` is passed, the validation or +filtering operation is simply appended to the lazy frame: -- {meth}`Schema.validate() ` -- {meth}`Schema.filter() ` -- {meth}`Collection.validate() ` -- {meth}`Collection.filter() ` +```python +def validate_lf(lf: pl.LazyFrame) -> dy.LazyFrame[MySchema]: + return lf.pipe(MySchema.validate) +``` + +For lazy inputs, validation is only run once the lazy frame is collected. If input data does not satisfy the schema, +no error is raised here, yet. + +## Collections: the `lazy` parameter -By default, `eager=True`. However, users may decide to set `eager=False` in order to simply append the validation or -the filtering operation to the lazy frame. For example, one might decide to run validation lazily: +{meth}`Collection.validate() ` and +{meth}`Collection.filter() ` collect their results by default, regardless of whether the +members are passed as data frames or lazy frames. Set `lazy=True` to keep the results lazy: ```python -def validate_lf(lf: pl.LazyFrame) -> pl.LazyFrame: - return lf.pipe(MySchema.validate, eager=False) +collection = MyCollection.validate(data, lazy=True) ``` -When `eager=False`, validation is only run once the lazy frame is collected. If input data does not satisfy the schema, -no error is raised here, yet. +Note that `lazy=True` cannot be used on a collection that defines at least one eager member, i.e. a member annotated +with `dy.DataFrame`. ## Error Types Due to current limitations in polars plugins, the type of error that is being raised from the `validate` function (both -for schemas and collections) is dependent on the value of the `eager` parameter: +for schemas and collections) is dependent on whether validation is performed eagerly or lazily: -- When `eager=True`, a {class}`~dataframely.ValidationError` is raised from the `validate` function -- When `eager=False`, a {class}`~polars.exceptions.ComputeError` is raised from the `collect` function +- When validating eagerly, a {class}`~dataframely.exc.ValidationError` is raised from the `validate` function +- When validating lazily, a {class}`~polars.exceptions.ComputeError` is raised from the `collect` function ```{note} For schemas, the error _message_ itself is equivalent. -For collections, the error message for `eager=False` is limited and non-deterministic: the error message only includes +For collections, the error message for lazy validation is limited and non-deterministic: the error message only includes information about a single member and, if multiple members fail validation, the member that the error message refers to may vary across executions. ``` @@ -51,7 +59,7 @@ in the error message and the specific failure surfaced may be non-deterministic By default, validation error messages report only the name of each failing rule and the number of rows that violated it. For easier debugging, dataframely can additionally include a few example rows for each failing rule. This is configured via {meth}`~dataframely.Config.set_max_failure_examples` (or the `max_failure_examples` keyword on the -{class}`~dataframely.Config` context manager) and applies to both `eager=True` and `eager=False`: +{class}`~dataframely.Config` context manager) and applies to both eager and lazy validation: ```python import dataframely as dy diff --git a/docs/guides/features/primary-keys.md b/docs/guides/features/primary-keys.md index c80d3e4b..6b19737d 100644 --- a/docs/guides/features/primary-keys.md +++ b/docs/guides/features/primary-keys.md @@ -42,6 +42,18 @@ class LineItemSchema(dy.Schema): Validation will now ensure that all pairs of (`invoice_id`, `item_id`) are unique. +### Unique columns + +In addition to the primary key, individual columns can be required to be unique by setting `unique=True`. Unlike +`primary_key=True`, setting `unique=True` on multiple columns does not form a composite constraint: each column is +checked for uniqueness independently. + +```python +class UserSchema(dy.Schema): + id = dy.String(primary_key=True) + email = dy.String(unique=True) +``` + ## Primary keys in {class}`~dataframely.Collection` The central idea behind {class}`~dataframely.Collection` is to unify multiple tables relating to the same set of diff --git a/docs/guides/features/sql-generation.md b/docs/guides/features/sql-generation.md index 563c2d17..bfa9ad75 100644 --- a/docs/guides/features/sql-generation.md +++ b/docs/guides/features/sql-generation.md @@ -64,15 +64,15 @@ df.write_database(connection=engine, table_name=my_table.name, if_table_exists=" ``` ```{note} -**Implementation:** The choice of `sqlalchemy` type is implemented in {meth}`~dataframely.Column.sqlalchemy_dtype`, which is overwritten by each of the subtypes of {class}`~dataframely.Column`. For example, the implementation for {class}`~dataframely.Date` is {meth}`~dataframely.Date.sqlalchemy_dtype`. +**Implementation:** The choice of `sqlalchemy` type is implemented in `Column.sqlalchemy_dtype`, which is overwritten by each of the subtypes of {class}`~dataframely.Column`. For example, the implementation for {class}`~dataframely.Date` is `Date.sqlalchemy_dtype`. ``` ```{note} -**Constraints:** The nullability and primary key constraints you define in `dataframely` are translated to SQL. Custom filters and rules are not. +**Constraints:** The nullability, primary key and `unique` constraints you define in `dataframely` are translated to SQL. Custom filters and rules are not. ``` ```{note} -**Length of string columns:** For string columns, `dataframely` will attempt to pass information about the maximal length into the SQL definition. This is trivial if `max_length` is set. Otherwise, if a `regex` is provided, +**Length of string columns:** For string columns, `dataframely` will attempt to pass information about the maximal length into the SQL definition. This is trivial if `max_length` is set. Otherwise, if a fully anchored `regex` (i.e. starting with `^` and ending with `$`) is provided, the maximal length of the string is inferred from the regular expression if possible. Note that having inferable maximal lengths can be particularly important for primary key columns. Some database systems, such as Microsoft SQL Server, do not allow `VARCHAR(max)` columns (unbounded strings) to be used as primary keys. ``` @@ -126,5 +126,5 @@ for name, dy_schema in MyCollection.member_schemas().items(): meta, *dy_schema.to_sqlalchemy_columns(dialect=engine.dialect), ) -meta.create_all() +meta.create_all(engine) ``` diff --git a/docs/guides/migration/v1-v2.md b/docs/guides/migration/v1-v2.md index aae8202c..3a78bb7b 100644 --- a/docs/guides/migration/v1-v2.md +++ b/docs/guides/migration/v1-v2.md @@ -12,7 +12,7 @@ Dataframely v2 finally implements lazy validation and filtering using a custom p ### Lazy `scan` operations -With lazy validation, all `scan_*` methods (e.g. {meth}`Schema.scan_parquet`, {meth}`Collection.scan_delta`, ...) are +With lazy validation, all `scan_*` methods (e.g. `Schema.scan_parquet`, `Collection.scan_delta`, ...) are now truly lazy, even if validation is necessary. Previously, this required collecting the input and running validation eagerly. diff --git a/docs/guides/quickstart.md b/docs/guides/quickstart.md index 428ffbfe..c0a2d02a 100644 --- a/docs/guides/quickstart.md +++ b/docs/guides/quickstart.md @@ -38,13 +38,15 @@ This translates into the following expectations on our data: ## Custom rules While parameters in the column-initializers allow for defining expectations on a single column (e.g. `min_length` for -string columns), this might not always by sufficient. +string columns), this might not always be sufficient. In many cases, we want to check expectations across columns: for example, the ratio between the number of bathrooms and bedrooms should not be too high. In `dataframely`, we can do this by adding a custom rule to our schema: ```python +import polars as pl + import dataframely as dy @@ -65,7 +67,30 @@ The decorator `@dy.rule()` "registers" the function as a rule using its name (i. The returned expression provides a boolean value for each row of the data which evaluates to `True` whenever the data are valid with respect to this rule. -Note that, for more complex rules that should evaluate across _rows_, you can simply use an `over` expression. +For defining even more complex rules that should evaluate across _rows_, you can simply use an `over` expression. +For our housing data, this allows us to specify, for example, that we want to observe at least two houses per zip code: + +```python +import polars as pl + +import dataframely as dy + + +class HouseSchema(dy.Schema): + zip_code = dy.String(nullable=False, min_length=3) + num_bedrooms = dy.UInt8(nullable=False) + num_bathrooms = dy.UInt8(nullable=False) + price = dy.Float64(nullable=False) + + @dy.rule() + def reasonable_bathroom_to_bedroom_ratio(cls) -> pl.Expr: + ratio = pl.col("num_bathrooms") / pl.col("num_bedrooms") + return (ratio >= 1 / 3) & (ratio <= 3) + + @dy.rule() + def minimum_zip_code_count(cls) -> pl.Expr: + return (pl.len() >= 2).over("zip_code") +``` ````{note} If you are using [`ruff`](https://docs.astral.sh/ruff/) to lint your code, you'll need to tell `ruff` to treat rules like classmethods. To this end, you can add the following to your `pyproject.toml`: @@ -101,17 +126,20 @@ validated_df = HouseSchema.validate(df, cast=True) If any row in `df` is invalid, i.e., any rule defined on individual columns or the entire schema evaluates to `False`, a validation exception is raised. -Here, we have invalid data in the `num_bedrooms` and `zip_code` columns. +Here, we have invalid data in the `num_bedrooms` and `zip_code` columns, and some rows violate the custom rules. - RuleValidationError: 2 rules failed validation: - * Column 'num_bedrooms' failed validation for 1 rules: - - 'nullability' failed for 2 rows - * Column 'zip_code' failed validation for 1 rules: - - 'min_length' failed for 1 rows + ValidationError: 4 rules failed validation: + - 'reasonable_bathroom_to_bedroom_ratio' failed for 1 rows + - 'minimum_zip_code_count' failed for 2 rows + * Column 'num_bedrooms' failed validation for 1 rules: + - 'nullability' failed for 2 rows + * Column 'zip_code' failed validation for 1 rules: + - 'min_length' failed for 1 rows Otherwise, if all rows in `df` are valid, `validate` returns a validated data frame of type `dy.DataFrame[HouseSchema]`. -The same applies when a `pl.LazyFrame` is passed to `validate`. +When a `pl.LazyFrame` is passed to `validate`, a `dy.LazyFrame[HouseSchema]` is returned instead and validation only +runs once the lazy frame is collected (see [lazy validation](features/lazy-validation.md)). The generic data frame types allow for more readable function signatures to express expectations on the schema of the data frame, e.g.: @@ -220,9 +248,8 @@ Lastly, `dataframely` schemas can be used to integrate with external tools: allowing to easily translate the dataframely schema into other packages' schema definitions: - `polars.Schema(HouseSchema)` can be used to construct a polars schema - `pyarrow.schema(HouseSchema)` can be used to construct a [pyarrow](https://arrow.apache.org/docs/python/index.html) schema -- `HouseSchema.to_sqlalchemy_columns()` provides a list of [sqlalchemy](https://www.sqlalchemy.org) columns that can be - used to - create SQL tables using types and constraints in line with the schema +- `HouseSchema.to_sqlalchemy_columns(dialect)` provides a list of [sqlalchemy](https://www.sqlalchemy.org) columns + that can be used to create SQL tables using types and constraints in line with the schema - You can use `dy.DataFrame[HouseSchema]` (or the `LazyFrame` equivalent) as fields in [pydantic](https://pydantic.dev) models, including support for validation and serialization. Integration with pydantic is unstable. diff --git a/skills/SKILL.md b/skills/SKILL.md index 03e2d913..8bef8a60 100644 --- a/skills/SKILL.md +++ b/skills/SKILL.md @@ -45,9 +45,9 @@ Persist all implicit assumptions on the data as constraints in the schema. Use d about the column contents. - Use the most specific type possible for each column (e.g. `dy.Enum` instead of `dy.String` when applicable). -- Use pre-defined arguments (e.g. `nullable`, `min`, `regex`) for column-level constraints if possible. +- Use pre-defined arguments (e.g. `nullable`, `unique`, `min`, `regex`) for column-level constraints if possible. - Use the `check` argument for non-standard column-level constraints that cannot be expressed using pre-defined - arguments. Prefer the defining the check as a dictionary with keys describing the type of check: + arguments. Prefer defining the check as a dictionary with keys describing the type of check: ```python class MySchema(dy.Schema): @@ -116,12 +116,14 @@ collection. streets: dy.LazyFrame[MyStreetSchema] @dy.filter() - def all_houses_on_known_streets(cls) -> pl.LazyFrame: + def all_houses_on_known_streets(self) -> pl.LazyFrame: return dy.functional.require_relationship_one_to_at_least_one( - cls.streets, cls.houses, on="street" + self.streets, self.houses, on="street" ) ``` + Filters receive an instance of the collection, so use `self` to refer to its members. + # Usage Conventions ## Clear Interfaces @@ -154,6 +156,8 @@ Both `.validate` and `.filter` enforce the schema at runtime. Pass `cast=True` f - `failure.details()` provides the data frame of invalid rows with additional columns providing information on which rules were violated +- **`Schema.is_valid`** — returns a boolean. Use when only the validity of a data frame is of interest. + When performing validation or filtering, prefer using `pipe` to clarify the flow of data: ```python @@ -161,6 +165,18 @@ result = df.pipe(MySchema.validate) out, failures = df.pipe(MySchema.filter) ``` +Validation and filtering follow polars' "eager-in-eager-out" and "lazy-in-lazy-out" semantics: passing a +`pl.DataFrame` returns eager results (and `.validate` raises immediately), passing a `pl.LazyFrame` returns lazy +results (and `.validate` only raises once the result is collected). Use lazy frames to let polars optimize validation +together with the surrounding query. + +### Collections + +`Collection.validate`, `Collection.filter` and `Collection.is_valid` work analogously, taking a mapping from member +names to data frames. They additionally enforce the collection's filters. `Collection.validate` and +`Collection.filter` collect their results by default; pass `lazy=True` to keep them lazy. `Collection.is_valid` always +evaluates eagerly and returns a boolean. `Collection.filter` returns one `FailureInfo` per member. + ### Pure Casting Use `Schema.cast` as an escape-hatch when it is already known that the data frame conforms to the schema and the @@ -188,19 +204,19 @@ from polars.testing import assert_frame_equal def test_grouped_sum(): df = pl.DataFrame( { - "col1": [1, 2, 3], - "col2": ["a", "a", "b"], + "group": ["a", "a", "b"], + "value": [1, 2, 3], } ).pipe(MyInputSchema.validate, cast=True) expected = pl.DataFrame( { - "col1": ["a", "b"], - "col2": [3, 3], + "group": ["a", "b"], + "value": [3, 3], } ) - result = my_code(df) + result = grouped_sum(df) assert_frame_equal(expected, result) ``` @@ -223,8 +239,11 @@ Synthetic data for collections should be generated as follows: - Use `MyCollection.sample(overrides=...)` to generate random data where certain values of the collection members matter. Use lists of dicts for providing overrides as "objects" spanning the collection members. - Values for shared primary keys must be provided at the root of the dictionaries - - Values for individual collection members must be provided in nested dictionaries under the keys corresponding to - the collection member names. + - Values for individual collection members must be provided under the keys corresponding to the collection member + names: as a nested dictionary for members sharing exactly the common primary key, and as a list of nested + dictionaries for members whose primary key extends the common primary key (i.e. multiple rows per sample). + - Values for members annotated with `inline_for_sampling=True` (via `dy.CollectionMember`) may be provided at the + root of the dictionaries. - Always use `MyCollection.create_empty()` instead of sampling with empty overrides when an empty collection is needed. # Getting more information