{ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Definition", "type": "object", "properties": { "workspace": { "description": "A workspace definition", "anyOf": [ { "$ref": "#/definitions/Workspace" }, { "type": "null" } ] }, "environment": { "description": "A environment definition", "anyOf": [ { "$ref": "#/definitions/Environment" }, { "type": "null" } ] }, "table": { "description": "A table definition", "anyOf": [ { "$ref": "#/definitions/Table" }, { "type": "null" } ] }, "classifier": { "description": "A classifier definition", "anyOf": [ { "$ref": "#/definitions/Classifier" }, { "type": "null" } ] }, "function": { "description": "A function definition", "anyOf": [ { "$ref": "#/definitions/Function" }, { "type": "null" } ] }, "config": { "description": "A workspace definition", "anyOf": [ { "$ref": "#/definitions/Config" }, { "type": "null" } ] }, "credential": { "description": "A credential definition", "anyOf": [ { "$ref": "#/definitions/Credential" }, { "type": "null" } ] } }, "additionalProperties": false, "definitions": { "Workspace": { "description": "A workspace is a collection of one or more catalogs, schemas, tables, and resources, called workspace members, that are managed together.", "type": "object", "required": [ "edition", "name" ], "properties": { "edition": { "description": "The SDF edition, should always be 1.2 (1.1 is deprecated)", "type": "string" }, "name": { "description": "The name of this workspace (defaults to the workspace directory name if not given) Name must be set for deployment.", "type": "string" }, "description": { "description": "A description of this workspace", "type": "string" }, "repository": { "description": "The URL of the workspace source repository (defaults to 'none' if no repository is given)", "type": "string" }, "includes": { "description": "An array of directories and filenames containing .sql and .sdf.yml files", "type": "array", "items": { "$ref": "#/definitions/IncludePath" } }, "excludes": { "description": "An array of directories and filenames to be skipped when resolving includes", "type": "array", "items": { "$ref": "#/definitions/ExcludePath" } }, "dependencies": { "description": "Dependencies of the workspace to other workspaces or to cloud database providers", "type": "array", "items": { "$ref": "#/definitions/Dependency" } }, "providers": { "description": "The providers for this environment", "type": "array", "items": { "$ref": "#/definitions/TargetProvider" } }, "integrations": { "description": "The integrations for this environment", "type": "array", "items": { "$ref": "#/definitions/Integration" } }, "defaults": { "description": "Defaults for this workspace", "anyOf": [ { "$ref": "#/definitions/Defaults" }, { "type": "null" } ] }, "source-locations": { "description": "Workspace defined by these set of files", "type": "array", "items": { "$ref": "#/definitions/FilePath" } }, "vars": { "description": "A map of named values for setting SQL variables from your environment Ex. -dt: dt, used in SQL as @dt, and in Jinja as \\{\\{ dt \\}\\}", "type": "object", "additionalProperties": { "$ref": "#/definitions/Constant" } } }, "additionalProperties": false }, "IncludePath": { "description": "All file path should either be relative to the workspace, or absolute for an object store like AWS s3:// Note that an [IncludePath] specifies a catalog and schema scope for unqualified names (in effect for both creating and querying tables). See [IncludePath::default_catalog] and [IncludePath::default_schema].", "type": "object", "required": [ "path" ], "properties": { "path": { "description": "A filepath", "type": "string" }, "type": { "description": "Type of included artifacts: model | test | stats | metadata | resource", "allOf": [ { "$ref": "#/definitions/IncludeType" } ] }, "index": { "description": "Index method for this include path: scan | table | schema-table | catalog-schema-table", "allOf": [ { "$ref": "#/definitions/IndexMethod" } ] }, "defaults": { "description": "Defaults for files on this path", "anyOf": [ { "$ref": "#/definitions/Defaults" }, { "type": "null" } ] } }, "additionalProperties": false }, "IncludeType": { "oneOf": [ { "description": "Models are .sql files (ddls or queries) used for data transformation. To learn more: https://docs.sdf.com/reference/sdf-yml#enum-includetype", "type": "string", "enum": [ "model" ] }, { "description": "Tests are expectations against the data of models. Uses the sdf-test library by default. To learn more: https://docs.sdf.com/reference/sdf-yml#enum-includetype", "type": "string", "enum": [ "test" ] }, { "description": "Checks are logical expectations expresses as queries against the information schema of the SDF workspace. To learn more: https://docs.sdf.com/reference/sdf-yml#enum-includetype", "type": "string", "enum": [ "check" ] }, { "description": "Reports are informational queries against the information schema of the SDF workspace. To learn more: https://docs.sdf.com/reference/sdf-yml#enum-includetype", "type": "string", "enum": [ "report" ] }, { "description": "Statistics are informational queries against the data of the models. To learn more: https://docs.sdf.com/reference/sdf-yml#enum-includetype", "type": "string", "enum": [ "stat" ] }, { "description": "Resources are local data files (like parquet, csv, or json) used by models in the workspace. To learn more: https://docs.sdf.com/reference/sdf-yml#enum-includetype", "type": "string", "enum": [ "resource" ] }, { "description": "Metadata are .sdf.yml files to add additional metadata. To learn more: https://docs.sdf.com/reference/sdf-yml#enum-includetype", "type": "string", "enum": [ "metadata" ] }, { "description": "Seeds are data files used both locally and in remote data warehouses. To learn more: https://docs.sdf.com/reference/sdf-yml#enum-includetype", "type": "string", "enum": [ "seed" ] } ] }, "IndexMethod": { "type": "string", "enum": [ "none", "table-name", "schema-table-name", "catalog-schema-table-name", "scan-dbt" ] }, "Defaults": { "type": "object", "properties": { "environment": { "description": "The default environment (can only be set on the level of the workspace)", "type": [ "string", "null" ] }, "dialect": { "description": "The dialect of this environment. If not set, defaults to trino", "anyOf": [ { "$ref": "#/definitions/Dialect" }, { "type": "null" } ] }, "compute": { "description": "DEPRECATED; subsumed by providers/integrations information", "anyOf": [ { "$ref": "#/definitions/ComputeKind" }, { "type": "null" } ] }, "preprocessor": { "description": "The preprocessor for this environment. If not set, defaults to local", "anyOf": [ { "$ref": "#/definitions/Preprocessor" }, { "type": "null" } ] }, "catalog": { "description": "Defines a default catalog. If not set, defaults to the (catalog/workspace) name in an outer scope", "type": [ "string", "null" ] }, "schema": { "description": "Defines a default schema, If not set, defaults to the schema name in an outer scope, if not set, defaults to 'pub'", "type": [ "string", "null" ] }, "materialization": { "description": "Defines the default materialization, if not set defaults to materialization in outer scope, if not set defaults to base-table", "anyOf": [ { "$ref": "#/definitions/Materialization" }, { "type": "null" } ] }, "creation-flag": { "description": "Defines table creation flags, defaults to if not set", "anyOf": [ { "$ref": "#/definitions/TableCreationFlags" }, { "type": "null" } ] }, "utils-lib": { "description": "The default utils library, if set overrides sdf_utils", "type": [ "string", "null" ] }, "test-lib": { "description": "The default test library, if set overrides sdf_test", "type": [ "string", "null" ] }, "materialization-lib": { "description": "The default materialization library, if set overrides sdf_materialization", "type": [ "string", "null" ] }, "index-method": { "description": "The default index for this tables", "anyOf": [ { "$ref": "#/definitions/IndexMethod" }, { "type": "null" } ] }, "include-type": { "description": "The default index for this tables", "anyOf": [ { "$ref": "#/definitions/IncludeType" }, { "type": "null" } ] }, "sync-method": { "description": "The default index for this tables", "anyOf": [ { "$ref": "#/definitions/SyncType" }, { "type": "null" } ] }, "severity": { "description": "The default severity for this tables tests and checks", "anyOf": [ { "$ref": "#/definitions/Severity" }, { "type": "null" } ] }, "csv-has-header": { "description": "CSV data has a header [only for external tables]", "type": [ "boolean", "null" ] }, "csv-delimiter": { "description": "CSV data is separated by this delimiter [only for external tables]", "type": [ "string", "null" ] }, "csv-compression": { "description": "Json or CSV data is compressed with this method [only for external tables]", "anyOf": [ { "$ref": "#/definitions/CompressionType" }, { "type": "null" } ] } }, "additionalProperties": false }, "Dialect": { "description": "Supported dialects", "type": "string", "enum": [ "snowflake", "presto", "bigquery", "redshift", "spark-lp" ] }, "ComputeKind": { "description": "Supported compute platforms", "type": "string", "enum": [ "local", "remote" ] }, "Preprocessor": { "type": "string", "enum": [ "none", "jinja", "sql-vars", "sql-logic-test", "all" ] }, "Materialization": { "type": "string", "enum": [ "table", "temporary-table", "external-table", "view", "materialized-view", "incremental-table", "snapshot-table", "recursive-table" ] }, "TableCreationFlags": { "oneOf": [ { "description": "Attempts to create a new table; fails if the table already exists.", "type": "string", "enum": [ "create-new" ] }, { "description": "Drops the existing table and creates a new one.", "type": "string", "enum": [ "drop-if-exists" ] }, { "description": "Skips table creation if the table already exists.", "type": "string", "enum": [ "skip-if-exists" ] }, { "description": "Replaces the existing table with a new one. Implementation may vary by DBMS.", "type": "string", "enum": [ "create-or-replace" ] }, { "description": "Creates the table only if it doesn't already exist.", "type": "string", "enum": [ "create-if-not-exists" ] } ] }, "SyncType": { "oneOf": [ { "description": "Synchronizes directory on pull and push", "type": "string", "enum": [ "always" ] }, { "description": "Synchronizes directory on every pull", "type": "string", "enum": [ "on-pull" ] }, { "description": "Synchronizes directory on every push", "type": "string", "enum": [ "on-push" ] }, { "description": "Never synchronizes directory", "type": "string", "enum": [ "never" ] } ] }, "Severity": { "description": "Supported dialects", "type": "string", "enum": [ "warning", "error" ] }, "CompressionType": { "description": "Compress table data using these methods", "oneOf": [ { "type": "string", "enum": [ "tar", "zstd" ] }, { "description": "BZIP2 Compression (.bz2)", "type": "string", "enum": [ "bzip2" ] }, { "description": "GZIP Compression (.gzip)", "type": "string", "enum": [ "gzip" ] }, { "description": "None, (default)", "type": "string", "enum": [ "none" ] } ] }, "ExcludePath": { "type": "object", "required": [ "path" ], "properties": { "path": { "description": "A filepath", "type": "string" }, "exclude-type": { "description": "Type of excluded artifacts", "anyOf": [ { "$ref": "#/definitions/ExcludeType" }, { "type": "null" } ] } }, "additionalProperties": false }, "ExcludeType": { "oneOf": [ { "type": "string", "enum": [ "content" ] }, { "description": "Excludes this path, can be a glob expression", "type": "string", "enum": [ "path" ] } ] }, "Dependency": { "type": "object", "required": [ "name" ], "properties": { "name": { "type": "string" }, "path": { "description": "The relative path from this workspace to the referenced workspace, for a Git repo, from the root of the depot to the workspace", "type": [ "string", "null" ] }, "environment": { "description": "The chosen workspace environment (none means default)", "type": [ "string", "null" ] }, "target": { "description": "The chosen workspace target (none means default)", "type": [ "string", "null" ] }, "git": { "description": "The Git repo", "type": [ "string", "null" ] }, "rev": { "description": "the Git revision (choose only one of the fields: rev, branch, tag)", "type": [ "string", "null" ] }, "branch": { "description": "the Git branch (choose only one of the fields: rev, branch, tag)", "type": [ "string", "null" ] }, "tag": { "description": "the Git tag (choose only one of the fields: rev, branch, tag)", "type": [ "string", "null" ] }, "imports": { "description": "Which models, reports, tests, checks etc. to include from the dependency", "type": "array", "items": { "type": "string" } } }, "additionalProperties": false }, "TargetProvider": { "description": "Table providers manage tables in in catalogs (OLD_PROVIDERS)", "type": "object", "required": [ "type" ], "properties": { "type": { "description": "The type of the catalog [e.g.: hive]", "allOf": [ { "$ref": "#/definitions/ProviderType" } ] }, "credential": { "description": "Credential identifier for this provider", "type": [ "string", "null" ] }, "warehouse": { "description": "The snowflake warehouse (defaults: the warehouse that was given at sdf Auth)", "type": [ "string", "null" ] }, "cluster-identifier": { "description": "The cluster identifier for redshift server", "writeOnly": true, "type": [ "string", "null" ] }, "batch-size": { "type": [ "integer", "null" ], "format": "uint", "minimum": 0.0 }, "sources": { "description": "A list of (possibly remote) sources to read, matched in order, so write specific pattern before more general patterns", "type": "array", "items": { "$ref": "#/definitions/SourcePattern" } }, "targets": { "description": "A list of (possibly remote) targets to build, matched in order, so write specific pattern before more general patterns, source patterns are excluded", "type": "array", "items": { "$ref": "#/definitions/TargetPattern" } }, "buckets": { "description": "A list o", "type": "array", "items": { "$ref": "#/definitions/DataBucket" } } }, "additionalProperties": false }, "ProviderType": { "type": "string", "enum": [ "glue", "redshift", "snowflake", "s3", "sdf" ] }, "SourcePattern": { "description": "Sources defines the tables that are possibly remote", "type": "object", "required": [ "pattern" ], "properties": { "pattern": { "description": "A source that can be read. Sources must be a three part names with globs, eg. *.*.* matches all catalogs, schema and table in scope", "type": "string" }, "preload": { "description": "Whether to preload the source", "type": [ "boolean", "null" ] }, "rename-from": { "description": "Renames sources when searching in the remote, the ith ${i} matches the ith * of the name, so to prepend all catalogs,schema,table with _, use \"_${1}._${2}._${3}\"", "type": [ "string", "null" ] } }, "additionalProperties": false }, "TargetPattern": { "description": "Sources define the tables that are possibly remote and that will be build", "type": "object", "properties": { "pattern": { "description": "A pattern must be a three part names with globs, eg. *.*.* matches all catalogs, schema and table in scope", "type": [ "string", "null" ] }, "patterns": { "description": "A list of patterns. A pattern must be a three part names with globs, eg. *.*.* matches all catalogs, schema and table in scope", "type": "array", "items": { "type": "string" } }, "preload": { "description": "Whether to preload the target", "type": [ "boolean", "null" ] }, "rename-as": { "description": "Renames targets, the ith ${i} matches the ith * of the name, so to prepend all catalogs,schema,table with _, use \"_${1}._${2}._${3}\"", "type": [ "string", "null" ] } }, "additionalProperties": false }, "DataBucket": { "type": "object", "required": [ "uri" ], "properties": { "uri": { "description": "The uri of the bucket", "type": "string" }, "region": { "description": "The region of the bucket", "type": [ "string", "null" ] } }, "additionalProperties": false }, "Integration": { "description": "Table providers manage tables in in catalogs (OLD_PROVIDERS)", "type": "object", "required": [ "provider" ], "properties": { "type": { "description": "The type of the integration [e.g.: database, metadata, data]", "default": "database", "allOf": [ { "$ref": "#/definitions/IntegrationType" } ] }, "provider": { "description": "The type of the provider [e.g.: snowflake, redshift, s3]", "allOf": [ { "$ref": "#/definitions/ProviderType" } ] }, "credential": { "description": "Credential identifier for this provider", "type": [ "string", "null" ] }, "cluster-identifier": { "description": "The cluster identifier for redshift server", "writeOnly": true, "type": [ "string", "null" ] }, "batch-size": { "description": "The size of the batch when querying the provider", "type": [ "integer", "null" ], "format": "uint", "minimum": 0.0 }, "sources": { "description": "A list of (possibly remote) sources to read, matched in order, so write specific pattern before more general patterns", "type": "array", "items": { "$ref": "#/definitions/SourcePattern" } }, "targets": { "description": "A list of (possibly remote) targets to build, matched in order, so write specific pattern before more general patterns, source patterns are excluded", "type": "array", "items": { "$ref": "#/definitions/TargetPattern" } }, "buckets": { "description": "A list o", "type": "array", "items": { "$ref": "#/definitions/DataBucket" } } }, "additionalProperties": false }, "IntegrationType": { "type": "string", "enum": [ "data", "metadata", "database" ] }, "FilePath": { "description": "All file path should either be relative to the workspace, or absolute for an object store like AWS s3://", "type": "object", "required": [ "path" ], "properties": { "path": { "description": "A filepath", "type": "string" }, "time": { "description": "Last modified of the file", "writeOnly": true, "anyOf": [ { "$ref": "#/definitions/SystemTime" }, { "type": "null" } ] } }, "additionalProperties": false }, "SystemTime": { "type": "object", "required": [ "nanos_since_epoch", "secs_since_epoch" ], "properties": { "secs_since_epoch": { "type": "integer", "format": "uint64", "minimum": 0.0 }, "nanos_since_epoch": { "type": "integer", "format": "uint32", "minimum": 0.0 } } }, "Constant": { "anyOf": [ { "type": "null" }, { "type": "boolean" }, { "type": "integer", "format": "int64" }, { "type": "number", "format": "double" }, { "type": "string" } ] }, "Environment": { "description": "Environments provide a way to override the fields of a workspace, i.e. if an environment has set field X, then the workspace field X will be overridden by the environment field X.", "type": "object", "required": [ "name" ], "properties": { "name": { "description": "The name of this workspace (defaults to the workspace directory name if not given) Name must be set for deployment.", "type": "string" }, "description": { "description": "A description of this workspace", "type": "string" }, "repository": { "description": "The URL of the workspace source repository (defaults to 'none' if no repository is given)", "type": "string" }, "includes": { "description": "An array of directories and filenames containing .sql and .sdf.yml files", "type": "array", "items": { "$ref": "#/definitions/IncludePath" } }, "excludes": { "description": "An array of directories and filenames to be skipped when resolving includes", "type": "array", "items": { "$ref": "#/definitions/ExcludePath" } }, "defaults": { "description": "Defaults for this workspace", "anyOf": [ { "$ref": "#/definitions/Defaults" }, { "type": "null" } ] }, "compute": { "description": "The compute platform for this environment. If not set, defaults to Local", "allOf": [ { "$ref": "#/definitions/ComputeKind" } ] }, "dependencies": { "description": "Dependencies of the workspace to other workspaces or to cloud database providers", "type": "array", "items": { "$ref": "#/definitions/Dependency" } }, "providers": { "description": "The providers for this environment", "type": "array", "items": { "$ref": "#/definitions/TargetProvider" } }, "integrations": { "description": "The integrations for this environment", "type": "array", "items": { "$ref": "#/definitions/Integration" } }, "vars": { "description": "A map of named values for setting SQL variables from your environment Ex. -dt: dt, used in SQL as @dt, and in Jinja as \\{\\{ dt \\}\\}", "type": "object", "additionalProperties": { "$ref": "#/definitions/Constant" } }, "source-locations": { "description": "Workspace defined by these set of files", "type": "array", "items": { "$ref": "#/definitions/FilePath" } }, "preprocessor": { "description": "Experimental: This project has jinja", "anyOf": [ { "$ref": "#/definitions/Preprocessor" }, { "type": "null" } ] } }, "additionalProperties": false }, "Table": { "description": "A tables is either defined by given data (also called external table) or defined via a query.", "type": "object", "required": [ "name" ], "properties": { "name": { "type": "string" }, "description": { "type": [ "string", "null" ] }, "dialect": { "description": "The dialect of this table, defaults to `presto`", "anyOf": [ { "$ref": "#/definitions/Dialect" }, { "type": "null" } ] }, "materialization": { "description": "The table-type of this table (new version)", "anyOf": [ { "$ref": "#/definitions/Materialization" }, { "type": "null" } ] }, "purpose": { "description": "Specify what kind of table or view this is", "anyOf": [ { "$ref": "#/definitions/TablePurpose" }, { "type": "null" } ] }, "origin": { "description": "The origin of this table or ", "anyOf": [ { "$ref": "#/definitions/TableOrigin" }, { "type": "null" } ] }, "exists-remotely": { "description": "Whether the table exists in the remote DB (used for is_incremental macto)", "type": [ "boolean", "null" ] }, "table-location": { "description": "Specify table ,location, defaults to none if not set", "anyOf": [ { "$ref": "#/definitions/TableLocation" }, { "type": "null" } ] }, "creation-flags": { "description": "Defines the table creation options, defaults to none if not set", "anyOf": [ { "$ref": "#/definitions/TableCreationFlags" }, { "type": "null" } ] }, "incremental-options": { "description": "Options governing incremental table evaluation (only for incremental tables)", "anyOf": [ { "$ref": "#/definitions/IncrementalOptions" }, { "type": "null" } ] }, "snapshot-options": { "description": "Options governing snapshot table evaluation (only for snapshot tables)", "anyOf": [ { "$ref": "#/definitions/SnapshotOptions" }, { "type": "null" } ] }, "dependencies": { "description": "All tables that this table depends on (syntax: catalog.schema.table)", "type": "array", "items": { "type": "string" } }, "depended-on-by": { "description": "All tables that depend on this table (syntax: catalog.schema.table)", "type": "array", "items": { "type": "string" } }, "columns": { "description": "The columns of the schema: name, type, metadata", "type": "array", "items": { "$ref": "#/definitions/Column" } }, "partitioned-by": { "description": "The partitioning format of the table", "type": "array", "items": { "$ref": "#/definitions/Partition" } }, "severity": { "description": "The default severity for this tables tests and checks", "anyOf": [ { "$ref": "#/definitions/Severity" }, { "type": "null" } ] }, "tests": { "type": "array", "items": { "$ref": "#/definitions/Constraint" } }, "schedule": { "description": "The schedule of the table [expressed as cron]", "type": "string" }, "starting": { "description": "The first date of the table [expressed by prefixes of RFC 33]", "type": "string" }, "classifiers": { "description": "An array of classifier references", "type": "array", "items": { "type": "string" } }, "reclassify": { "description": "Array of reclassify instructions for changing the attached classifier labels", "type": "array", "items": { "$ref": "#/definitions/Reclassify" } }, "lineage": { "description": "Lineage, a tagged array of column references", "anyOf": [ { "$ref": "#/definitions/Lineage" }, { "type": "null" } ] }, "location": { "description": "Data is at this location", "type": [ "string", "null" ] }, "file-format": { "description": "Store table in this format [only for external tables]", "anyOf": [ { "$ref": "#/definitions/FileFormat" }, { "type": "null" } ] }, "with-header": { "description": "CSV data has a header [only for external tables]", "type": [ "boolean", "null" ] }, "delimiter": { "description": "CSV data is separated by this delimiter [only for external tables]", "type": [ "string", "null" ] }, "compression": { "description": "Json or CSV data is compressed with this method [only for external tables]", "anyOf": [ { "$ref": "#/definitions/CompressionType" }, { "type": "null" } ] }, "seeded": { "description": "Is this table backed by a DDL or a type provider", "type": [ "boolean", "null" ] }, "source-locations": { "description": "Table is defined by these .sql and/or .sdf files", "type": "array", "items": { "$ref": "#/definitions/FilePath" } }, "sealed": { "description": "This table is either backed by a create table ddl or by a table definition in yml that is the table's complete schema", "type": [ "boolean", "null" ] } }, "additionalProperties": false }, "TablePurpose": { "oneOf": [ { "type": "string", "enum": [ "report" ] }, { "description": "A regular table", "type": "string", "enum": [ "model" ] }, { "description": "A code contract", "type": "string", "enum": [ "check" ] }, { "description": "A data contract", "type": "string", "enum": [ "test" ] }, { "description": "A data report", "type": "string", "enum": [ "stat" ] }, { "description": "A SDF System table/view, maintained by the sdf cli", "type": "string", "enum": [ "system" ] }, { "description": "An External System table/view, maintained by the database system (e.g. Snowflake, Redshift, etc.)", "type": "string", "enum": [ "external-system" ] } ] }, "TableOrigin": { "type": "string", "enum": [ "local", "remote" ] }, "TableLocation": { "type": "string", "enum": [ "mirror", "local", "remote", "intrinsic" ] }, "IncrementalOptions": { "type": "object", "required": [ "strategy" ], "properties": { "strategy": { "description": "Incremental strategy; may be one of `append`, `merge`, or `delete+insert`", "allOf": [ { "$ref": "#/definitions/IncrementalStrategy" } ] }, "unique-key": { "description": "Expression used for identifying records in Merge and Delete+Insert strategies; May be a column name or an expression combining multiple columns. If left unspecified, Merge and Delete+Insert strategies behave the same as Append", "type": [ "string", "null" ] }, "merge-update-columns": { "description": "List of column names to be updated as part of Merge strategy; Only one of merge_update_columns or merge_exclude_columns may be specified", "type": "array", "items": { "type": "string" } }, "merge-exclude-columns": { "description": "List of column names to exclude from updating as part of Merge strategy; Only one of merge_update_columns or merge_exclude_columns may be specified", "type": "array", "items": { "type": "string" } }, "on-schema-change": { "description": "Method for reacting to changing schema in the source of the incremental table Possible values are `fail`, `append`, and `sync`. If left unspecified, the default behavior is to ignore the change and possibly error out if the schema change is incompatible. `fail` causes a failure whenever any deviation in the schema of the source is detected; `append` adds new columns but does not delete the columns removed from the source; `sync` adds new columns and deletess the columns removed from the source;", "anyOf": [ { "$ref": "#/definitions/OnSchemaChange" }, { "type": "null" } ] } }, "additionalProperties": false }, "IncrementalStrategy": { "type": "string", "enum": [ "append", "merge", "delete+insert" ] }, "OnSchemaChange": { "oneOf": [ { "description": "Fail when the new schema of an incremental table does not match the old one", "type": "string", "enum": [ "fail" ] }, { "description": "Keep all the columns of the old incremental table schema while appending the new columns from the new schema. Initialize all the columns that were deleted in the new schema with NULL", "type": "string", "enum": [ "append" ] }, { "description": "Delete all the deleted columns from the incremental table; Append all the new columns", "type": "string", "enum": [ "sync" ] } ] }, "SnapshotOptions": { "type": "object", "required": [ "strategy", "unique-key" ], "properties": { "strategy": { "description": "Snapshot strategy; may be one of `timestamp` (default), or `check`", "allOf": [ { "$ref": "#/definitions/SnapshotStrategy" } ] }, "unique-key": { "description": "Expression used for identifying records that will be updated according to the snapshot strategiy; May be a column name or an expression combining multiple columns", "type": "string" }, "updated-at": { "description": "Name of the timestamp column used to identify the last update time This option is only required for the `timestamp` snapshot strategy", "type": [ "string", "null" ] }, "check-cols": { "description": "Specification of which columns to check for change (may be a list of column names or `all`) This option is only required for the `check` snapshot strategy", "anyOf": [ { "$ref": "#/definitions/CheckColsSpec" }, { "type": "null" } ] } }, "additionalProperties": false }, "SnapshotStrategy": { "type": "string", "enum": [ "timestamp", "check" ] }, "CheckColsSpec": { "oneOf": [ { "description": "A list of column names to be used for checking if a snapshot's source data set was updated", "type": "object", "required": [ "cols" ], "properties": { "cols": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false }, { "description": "Use all columns to check whether a snapshot's source data set was updated", "type": "string", "enum": [ "all" ] } ] }, "Column": { "type": "object", "required": [ "name" ], "properties": { "name": { "description": "The name of the column", "type": "string" }, "description": { "description": "A description of this column", "type": "string" }, "datatype": { "description": "The type of this column", "type": [ "string", "null" ] }, "classifiers": { "description": "An array of classifier references", "type": "array", "items": { "type": "string" } }, "lineage": { "description": "Lineage, a tagged array of column references", "anyOf": [ { "$ref": "#/definitions/Lineage" }, { "type": "null" } ] }, "forward-lineage": { "description": "Forward Lineage, the columns that this column is used to compute", "anyOf": [ { "$ref": "#/definitions/Lineage" }, { "type": "null" } ] }, "reclassify": { "description": "Array of reclassify instructions for changing the attached classifier labels", "type": "array", "items": { "$ref": "#/definitions/Reclassify" } }, "samples": { "description": "An array of representative literals of this column [experimental!]", "type": "array", "items": { "type": "string" } }, "default-severity": { "description": "The default severity for this tables tests and checks", "allOf": [ { "$ref": "#/definitions/Severity" } ] }, "tests": { "type": "array", "items": { "$ref": "#/definitions/Constraint" } } }, "additionalProperties": false }, "Lineage": { "type": "object", "properties": { "copy": { "description": "The output column is computed by copying these upstream columns", "type": "array", "items": { "type": "string" } }, "modify": { "description": "The output column is computed by transforming these upstream columns", "type": "array", "items": { "type": "string" } }, "scan": { "description": "These upstream columns are indirectly used to produce the output (e.g. in WHERE or GROUP BY)", "type": "array", "items": { "type": "string" } }, "apply": { "description": "These functions were used to produce the output column", "type": "array", "items": { "type": "string" } } }, "additionalProperties": false }, "Reclassify": { "type": "object", "properties": { "to": { "description": "Target classifier", "type": [ "string", "null" ] }, "from": { "description": "Expected source classifier", "type": [ "string", "null" ] } }, "additionalProperties": false }, "Constraint": { "type": "object", "required": [ "expect" ], "properties": { "expect": { "description": "The constraint macro: must have the form lib.macro(args,..), where lib is any of the libs in scope, std is available by default", "type": "string" }, "severity": { "description": "The severity of this constraint", "anyOf": [ { "$ref": "#/definitions/Severity" }, { "type": "null" } ] } }, "additionalProperties": false }, "Partition": { "description": "A partition is a table column, used to describe to which partition this row belongs to", "type": "object", "required": [ "name" ], "properties": { "name": { "description": "The name of the partition column", "type": "string" }, "description": { "description": "A description of the partition column", "type": [ "string", "null" ] }, "format": { "description": "The format of the partition column [use strftime format for date/time] See (guide)[https://docs.sdf.com/guide/schedules]", "type": [ "string", "null" ] } }, "additionalProperties": false }, "FileFormat": { "description": "Store table data in these formats", "type": "string", "enum": [ "parquet", "csv", "json" ] }, "Classifier": { "description": "A classifier defines the labels that can be attached to columns or a table.", "type": "object", "required": [ "name" ], "properties": { "name": { "description": "The name of the classifier type", "type": "string" }, "description": { "description": "A description of this classifier type", "type": "string" }, "labels": { "description": "Named classifier labels", "type": "array", "items": { "$ref": "#/definitions/Label" } }, "scope": { "description": "Scope of the classifier: table or column", "allOf": [ { "$ref": "#/definitions/Scope" } ] }, "cardinality": { "description": "Cardinality of the classifier: zero-or-one, one or zero-or-many", "allOf": [ { "$ref": "#/definitions/Cardinality" } ] }, "propagate": { "description": "Does the classifier propagate from scope to scope or is it a one scope marker", "type": "boolean" }, "source-locations": { "description": "Classifier defined by these set of .sdf files", "type": "array", "items": { "$ref": "#/definitions/FilePath" } } }, "additionalProperties": false }, "Label": { "description": "A classifier element is a scoped classifier label (e.g. the element PII belongs to the classifier scope data)", "type": "object", "required": [ "name" ], "properties": { "name": { "description": "The name of the label, use \"*\" to allow arbitrary strings as labels", "type": "string" }, "description": { "description": "A description of this classifier element", "type": [ "string", "null" ] } }, "additionalProperties": false }, "Scope": { "type": "string", "enum": [ "column", "table" ] }, "Cardinality": { "type": "string", "enum": [ "zero-or-one", "one", "zero-or-more" ] }, "Function": { "description": "A function block defines the signature for user defined", "type": "object", "required": [ "name" ], "properties": { "name": { "description": "The name of the function [syntax: [[catalog.]schema].function]", "type": "string" }, "section": { "description": "The generic type bounds", "type": "string" }, "dialect": { "description": "The dialect that provides this function", "anyOf": [ { "$ref": "#/definitions/Dialect" }, { "type": "null" } ] }, "description": { "description": "A description of this function", "type": "string" }, "variadic": { "description": "Arbitrary number of arguments of an common type out of a list of valid types", "allOf": [ { "$ref": "#/definitions/Variadic" } ] }, "kind": { "description": "The function kind", "allOf": [ { "$ref": "#/definitions/FunctionKind" } ] }, "parameters": { "description": "The arguments of this function", "type": [ "array", "null" ], "items": { "$ref": "#/definitions/Parameter" } }, "optional-parameters": { "description": "The arguments of this function", "type": [ "array", "null" ], "items": { "$ref": "#/definitions/OptionalParameter" } }, "returns": { "description": "The results of this function (can be a tuple)", "anyOf": [ { "$ref": "#/definitions/Parameter" }, { "type": "null" } ] }, "binds": { "description": "The generic type bounds", "type": "array", "items": { "$ref": "#/definitions/TypeBound" } }, "volatility": { "description": "volatility - The volatility of the function.", "allOf": [ { "$ref": "#/definitions/Volatility" } ] }, "examples": { "description": "example - Example use of the function (tuple with input/output)", "type": "array", "items": { "$ref": "#/definitions/Example" } }, "cross-link": { "description": "cross-link - link to existing documentation, for example: https://prestodb.io/docs/current/functions/datetime.html#truncation-function", "type": "string" }, "reclassify": { "description": "Array of reclassify instructions for changing the attached classifier labels", "type": "array", "items": { "$ref": "#/definitions/Reclassify" } }, "source-locations": { "description": "Function defined by these set of .sdf files", "type": "array", "items": { "$ref": "#/definitions/FilePath" } }, "implemented-by": { "anyOf": [ { "$ref": "#/definitions/FunctionImplSpec" }, { "type": "null" } ] }, "special": { "description": "Function can be called without parentheses, e.g. as if it were a constant, e.g. current_date", "type": "boolean" } }, "additionalProperties": false }, "Variadic": { "description": "Arbitrary number of arguments of an common type out of a list of valid types", "oneOf": [ { "type": "string", "enum": [ "non-uniform" ] }, { "description": "All arguments have the same types", "type": "string", "enum": [ "uniform" ] }, { "description": "All even arguments have one type, odd arguments have another type", "type": "string", "enum": [ "even-odd" ] }, { "description": "Any length of arguments, arguments can be different types", "type": "string", "enum": [ "any" ] } ] }, "FunctionKind": { "type": "string", "enum": [ "scalar", "aggregate", "window", "table" ] }, "Parameter": { "description": "A function parameter", "type": "object", "properties": { "name": { "description": "The name of the parameter", "type": [ "string", "null" ] }, "description": { "description": "A description of this parameter", "type": [ "string", "null" ] }, "datatype": { "description": "The datatype of this parameter", "type": [ "string", "null" ] }, "classifier": { "description": "An array of classifier references", "type": [ "array", "null" ], "items": { "type": "string" } }, "constant": { "description": "The required constant value of this parameter", "type": [ "string", "null" ] }, "identifiers": { "description": "The parameter may appear as identifier, without quote", "type": [ "array", "null" ], "items": { "type": "string" } } }, "additionalProperties": false }, "OptionalParameter": { "description": "A function parameter", "type": "object", "required": [ "datatype", "name" ], "properties": { "name": { "description": "The name of the parameter", "type": "string" }, "description": { "description": "A description of this parameter", "type": [ "string", "null" ] }, "datatype": { "description": "The datatype of this parameter", "type": "string" }, "classifier": { "description": "An array of classifier references", "type": [ "array", "null" ], "items": { "type": "string" } }, "constant": { "description": "The required constant value of this parameter", "type": [ "string", "null" ] }, "identifiers": { "description": "The parameter may appear as identifier, without quote", "type": [ "array", "null" ], "items": { "type": "string" } } }, "additionalProperties": false }, "TypeBound": { "type": "object", "required": [ "datatypes", "type-variable" ], "properties": { "type-variable": { "type": "string" }, "datatypes": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false }, "Volatility": { "description": "A function's volatility, which defines the functions eligibility for certain optimizations", "oneOf": [ { "description": "Pure - An pure function will always return the same output when given the same input.", "type": "string", "enum": [ "pure" ] }, { "description": "Stable - A stable function may return different values given the same input across different queries but must return the same value for a given input within a query.", "type": "string", "enum": [ "stable" ] }, { "description": "Volatile - A volatile function may change the return value from evaluation to evaluation. Multiple invocations of a volatile function may return different results when used in the same query.", "type": "string", "enum": [ "volatile" ] } ] }, "Example": { "type": "object", "required": [ "input", "output" ], "properties": { "input": { "description": "The sql string corresponding to the input of this example", "type": "string" }, "output": { "description": "The output corresponding to running the input string", "type": "string" } }, "additionalProperties": false }, "FunctionImplSpec": { "description": "Indicates how a function's evaluation is implemented.", "oneOf": [ { "description": "By a built-in primitive in Datafusion. (Being phased out in favor of UDFs.)", "type": "string", "enum": [ "builtin" ] }, { "description": "By a UDF in the sdf-functions crate.", "type": "object", "required": [ "rust" ], "properties": { "rust": { "$ref": "#/definitions/RustFunctionSpec" } }, "additionalProperties": false }, { "description": "By a UDF in the datafusion crate.", "type": "object", "required": [ "datafusion" ], "properties": { "datafusion": { "$ref": "#/definitions/DataFusionSpec" } }, "additionalProperties": false }, { "description": "By a CREATE FUNCTION. (Not yet supported for evaluation.)", "type": "string", "enum": [ "sql" ] } ] }, "RustFunctionSpec": { "type": "object", "properties": { "name": { "description": "The name attribute of the implementing UDF. None indicates the UDF is named the same as the function.", "type": [ "string", "null" ] } }, "additionalProperties": false }, "DataFusionSpec": { "type": "object", "properties": { "udf": { "description": "The name attribute of the implementing UDF. None indicates the UDF is named the same as the function.", "type": [ "string", "null" ] } }, "additionalProperties": false }, "Config": { "description": "A configuration with section name and properties", "type": "object", "required": [ "name" ], "properties": { "name": { "description": "The name of the configuration section", "type": "string" }, "description": { "description": "A description of this configuration section", "type": [ "string", "null" ] }, "properties": { "type": [ "object", "null" ], "additionalProperties": { "type": "string" } } }, "additionalProperties": false }, "Credential": { "type": "object", "oneOf": [ { "type": "object", "required": [ "type", "variant" ], "properties": { "type": { "type": "string", "enum": [ "sdf" ] }, "variant": { "description": "Variant of the credential", "allOf": [ { "$ref": "#/definitions/SdfAuthVariant" } ] }, "headless-creds": { "description": "Headless Credentials", "anyOf": [ { "$ref": "#/definitions/HeadlessCredentials" }, { "type": "null" } ] } } }, { "type": "object", "required": [ "account-id", "password", "type", "username" ], "properties": { "type": { "type": "string", "enum": [ "snowflake" ] }, "account-id": { "description": "The account id of your snowflake account (e.g. . or )", "type": "string" }, "username": { "description": "The user name to connect to snowflake", "type": "string" }, "password": { "description": "The password to connect to snowflake", "type": "string" }, "role": { "description": "The role to use for the connection", "type": [ "string", "null" ] }, "warehouse": { "description": "The warehouse to use for the connection", "type": [ "string", "null" ] } } }, { "type": "object", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ "aws" ] }, "profile": { "description": "The name of th profile to use in the AWS credentials file", "type": [ "string", "null" ] }, "default-region": { "description": "The default region to use for this profile", "type": [ "string", "null" ] }, "role-arn": { "description": "The arn of the role to assume", "type": [ "string", "null" ] }, "external-id": { "description": "The external id to use for the role", "type": [ "string", "null" ] }, "use-web-identity": { "description": "Whether to use a web identity for authentication", "type": [ "boolean", "null" ] }, "access-key-id": { "description": "The access key id to use for the connection", "type": [ "string", "null" ] }, "secret-access-key": { "description": "The secret access key to use for the connection", "type": [ "string", "null" ] }, "session-token": { "description": "The session token to use for the connection", "type": [ "string", "null" ] } } }, { "type": "object", "required": [ "api-key", "type" ], "properties": { "type": { "type": "string", "enum": [ "openai" ] }, "api-key": { "description": "The api key to use for the connection", "type": "string" } } }, { "type": "object", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ "empty" ] } } } ], "required": [ "name", "source-locations" ], "properties": { "name": { "description": "The name of the credential (default = 'default')", "type": "string" }, "description": { "description": "A description of this credential", "type": [ "string", "null" ] }, "source-locations": { "description": "Credential defined by these set of .sdf files", "writeOnly": true, "type": "array", "items": { "$ref": "#/definitions/FilePath" } } } }, "SdfAuthVariant": { "type": "string", "enum": [ "interactive", "headless", "id_token_from_file" ] }, "HeadlessCredentials": { "type": "object", "required": [ "access_key", "secret_key" ], "properties": { "access_key": { "type": "string" }, "secret_key": { "type": "string" } } } } }