commands =   !windows: /bin/bash -c '! pip list --exclude-editable | grep -e dagster -e dagit'   coverage erase   echo -e "--- \033[0;32m:pytest: Running tox tests\033[0m"+  python -m pip install "dask[dataframe]" --upgrade+  pip install pyarrow  

+import dask.dataframe as dd++from dagster import (+    Any,+    AssetMaterialization,+    Bool,+    DagsterInvariantViolationError,+    DagsterType,+    Enum,+    EnumValue,+    EventMetadataEntry,+    Field,+    Int,+    Permissive,+    String,+    TypeCheck,+    check,+    dagster_type_loader,+    dagster_type_materializer,+)+from dagster.config.field_utils import Selector++WriteCompressionTextOptions = Enum(+    'WriteCompressionText', [EnumValue('gzip'), EnumValue('bz2'), EnumValue('xz'),],+)++EngineParquetOptions = Enum(+    'EngineParquet', [EnumValue('auto'), EnumValue('fastparquet'), EnumValue('pyarrow'),],+)+++def dict_without_keys(ddict, *keys):+    return {key: value for key, value in ddict.items() if key not in set(keys)}+++@dagster_type_materializer(+    Selector(+        {+            'csv': Permissive(+                {+                    'path': Field(+                        Any,+                        is_required=True,+                        description="str or list, Path glob indicating the naming scheme for the output files",+                    ),+                    'single_file': Field(+                        Bool,+                        is_required=False,+                        description="Whether to save everything into a single CSV file. Under the single file mode, each partition is appended at the end of the specified CSV file. Note that not all filesystems support the append mode and thus the single file mode, especially on cloud storage systems such as S3 or GCS. A warning will be issued when writing to a file that is not backed by a local filesystem.",+                    ),+                    'encoding': Field(+                        String,+                        is_required=False,+                        description="A string representing the encoding to use in the output file, defaults to 'ascii' on Python 2 and 'utf-8' on Python 3.",+                    ),+                    'mode': Field(+                        String, is_required=False, description="Python write mode, default 'w'",+                    ),+                    'compression': Field(+                        WriteCompressionTextOptions,+                        is_required=False,+                        description="a string representing the compression to use in the output file, allowed values are 'gzip', 'bz2', 'xz'",+                    ),+                    'compute': Field(+                        Bool,+                        is_required=False,+                        description="If true, immediately executes. If False, returns a set of delayed objects, which can be computed at a later time.",+                    ),+                    'storage_options': Field(+                        Permissive(),+                        is_required=False,+                        description="Parameters passed on to the backend filesystem class.",+                    ),+                    'header_first_partition_only': Field(+                        Bool,+                        is_required=False,+                        description="If set to `True`, only write the header row in the first output file. By default, headers are written to all partitions under the multiple file mode (`single_file` is `False`) and written only once under the single file mode (`single_file` is `True`). It must not be `False` under the single file mode.",+                    ),+                    'compute_kwargs': Field(+                        Permissive(),+                        is_required=False,+                        description="Options to be passed in to the compute method",+                    ),+                }+            ),+            'parquet': Permissive(+                {+                    'path': Field(+                        Any,+                        is_required=True,+                        description="str or pathlib.Path, Destination directory for data. Prepend with protocol like ``s3://`` or ``hdfs://`` for remote data.",+                    ),+                    'engine': Field(+                        EngineParquetOptions,+                        is_required=False,+                        description="{'auto', 'fastparquet', 'pyarrow'}, default 'auto' Parquet library to use. If only one library is installed, it will use that one; if both, it will use 'fastparquet'.",+                    ),+                    'compression': Field(+                        Any,+                        is_required=False,+                        description="str or dict, optional Either a string like ``'snappy'`` or a dictionary mapping column names to compressors like ``{'name': 'gzip', 'values': 'snappy'}``. The default is ``'default'``, which uses the default compression for whichever engine is selected.",+                    ),+                    'write_index': Field(+                        Bool,+                        is_required=False,+                        description="Whether or not to write the index. Defaults to True.",+                    ),+                    'append': Field(+                        Bool,+                        is_required=False,+                        description="If False (default), construct data-set from scratch. If True, add new row-group(s) to an existing data-set. In the latter case, the data-set must exist, and the schema must match the input data.",+                    ),+                    'ignore_divisions': Field(+                        Bool,+                        is_required=False,+                        description="If False (default) raises error when previous divisions overlap with the new appended divisions. Ignored if append=False.",+                    ),+                    'partition_on': Field(+                        list,+                        is_required=False,+                        description="Construct directory-based partitioning by splitting on these fields values. Each dask partition will result in one or more datafiles, there will be no global groupby.",+                    ),+                    'storage_options': Field(+                        Permissive(),+                        is_required=False,+                        description="Key/value pairs to be passed on to the file-system backend, if any.",+                    ),+                    'write_metadata_file': Field(+                        Bool,+                        is_required=False,+                        description="Whether to write the special '_metadata' file.",+                    ),+                    'compute': Field(+                        Bool,+                        is_required=False,+                        description="If True (default) then the result is computed immediately. If False then a ``dask.delayed`` object is returned for future computation.",+                    ),+                    'compute_kwargs': Field(+                        Permissive(),+                        is_required=False,+                        description="Options to be passed in to the compute method.",+                    ),+                }+            ),+            'hdf': Permissive(+                {+                    'path': Field(+                        Any,+                        is_required=True,+                        description="str or pathlib.Path, Path to a target filename. Supports strings, ``pathlib.Path``, or any object implementing the ``__fspath__`` protocol. May contain a ``*`` to denote many filenames.",+                    ),+                    'key': Field(+                        String,+                        is_required=True,+                        description="Datapath within the files. May contain a ``*`` to denote many locations",+                    ),+                    'compute': Field(+                        Bool,+                        is_required=False,+                        description="Whether or not to execute immediately.  If False then this returns a ``dask.Delayed`` value.",+                    ),+                    'scheduler': Field(+                        String,+                        is_required=False,+                        description="The scheduler to use, like 'threads' or 'processes'",+                    ),+                }+            ),+            'json': Permissive(+                {+                    'path': Field(+                        Any,+                        is_required=True,+                        description="str or list, Location to write to. If a string, and there are more than one partitions in df, should include a glob character to expand into a set of file names, or provide a ``name_function=`` parameter. Supports protocol specifications such as ``'s3://'``.",+                    ),+                    'encoding': Field(+                        String,+                        is_required=False,+                        description="default is 'utf-8', The text encoding to implement, e.g., 'utf-8'",+                    ),+                    'errors': Field(+                        String,+                        is_required=False,+                        description="default is 'strict', how to respond to errors in the conversion (see ``str.encode()``)",+                    ),+                    'storage_options': Field(+                        Permissive(),+                        is_required=False,+                        description="Passed to backend file-system implementation",+                    ),+                    'compute': Field(+                        Bool,+                        is_required=False,+                        description="If true, immediately executes. If False, returns a set of delayed objects, which can be computed at a later time.",+                    ),+                    'compute_kwargs': Field(+                        Permissive(),+                        is_required=False,+                        description="Options to be passed in to the compute method",+                    ),+                    'compression': Field(+                        String, is_required=False, description="String like 'gzip' or 'xz'.",+                    ),+                },+            ),+            'sql': Permissive(+                {+                    'name': Field(String, is_required=True, description="Name of SQL table",),+                    'uri': Field(+                        String,+                        is_required=True,+                        description="Full sqlalchemy URI for the database connection",+                    ),+                    'schema': Field(+                        String,+                        is_required=False,+                        description="Specify the schema (if database flavor supports this). If None, use default schema.",+                    ),+                    'if_exists': Field(+                        String,+                        is_required=False,+                        description="""+                            {'fail', 'replace', 'append'}, default 'fail'"+                            How to behave if the table already exists.+                            * fail: Raise a ValueError.+                            * replace: Drop the table before inserting new values.+                            * append: Insert new values to the existing table.+                        """,+                    ),+                    'index': Field(+                        Bool,+                        is_required=False,+                        description="default is True, Write DataFrame index as a column. Uses `index_label` as the column name in the table.",+                    ),+                    'index_label': Field(+                        Any,+                        is_required=False,+                        description="str or sequence, default None Column label for index column(s). If None is given (default) and `index` is True, then the index names are used. A sequence should be given if the DataFrame uses MultiIndex.",+                    ),+                    'chunksize': Field(+                        Int,+                        is_required=False,+                        description="Specify the number of rows in each batch to be written at a time. By default, all rows will be written at once",+                    ),+                    'dtype': Field(+                        Any,+                        is_required=False,+                        description="dict or scalar, Specifying the datatype for columns. If a dictionary is used, the keys should be the column names and the values should be the SQLAlchemy types or strings for the sqlite3 legacy mode. If a scalar is provided, it will be applied to all columns.",+                    ),+                    'method': Field(+                        String,+                        is_required=False,+                        description="""+                            {None, 'multi', callable}, default None+                            Controls the SQL insertion clause used:+                            * None : Uses standard SQL ``INSERT`` clause (one per row).+                            * 'multi': Pass multiple values in a single ``INSERT`` clause.+                            * callable with signature ``(pd_table, conn, keys, data_iter)``.+                            Details and a sample callable implementation can be found in the+                            section :ref:`insert method <io.sql.method>`.+                        """,+                    ),+                    'compute': Field(+                        Bool,+                        is_required=False,+                        description="default is True, When true, call dask.compute and perform the load into SQL; otherwise, return a Dask object (or array of per-block objects when parallel=True)",+                    ),+                    'parallel': Field(+                        Bool,+                        is_required=False,+                        description="default is False, When true, have each block append itself to the DB table concurrently. This can result in DB rows being in a different order than the source DataFrame's corresponding rows. When false, load each block into the SQL DB in sequence.",+                    ),+                },+            ),+        },+    )+)+def dataframe_materializer(_context, config, dask_df):+    check.inst_param(dask_df, 'dask_df', dd.DataFrame)+    file_type, file_options = list(config.items())[0]+    path = file_options.get('path')++    if file_type == 'csv':+        dask_df.to_csv(path, **dict_without_keys(file_options, 'path'))+    elif file_type == 'parquet':+        dask_df.to_parquet(path, **dict_without_keys(file_options, 'path'))+    elif file_type == 'hdf':+        dask_df.to_hdf(path, **dict_without_keys(file_options, 'path'))+    elif file_type == 'json':+        dask_df.to_json(path, **dict_without_keys(file_options, 'path'))+    elif file_type == 'sql':+        dask_df.to_sql(**file_options)+    else:+        check.failed('Unsupported file_type {file_type}'.format(file_type=file_type))++    return AssetMaterialization.file(path)+++@dagster_type_loader(+    Selector(+        {+            'csv': Permissive(+                {+                    'path': Field(+                        Any,+                        is_required=True,+                        description="str or list, Absolute or relative filepath(s). Prefix with a protocol like `s3://` to read from alternative filesystems. To read from multiple files you can pass a globstring or a list of paths, with the caveat that they must all have the same protocol.",+                    ),+                    'blocksize': Field(+                        Any,+                        is_required=False,+                        description="str or int or None, Number of bytes by which to cut up larger files. Default value is computed based on available physical memory and the number of cores, up to a maximum of 64MB. Can be a number like 64000000` or a string like ``'64MB'. If None, a single block is used for each file.",+                    ),+                    'sample': Field(+                        Int,+                        is_required=False,+                        description="Number of bytes to use when determining dtypes.",+                    ),+                    'assume_missing': Field(+                        Bool,+                        is_required=False,+                        description="If True, all integer columns that aren’t specified in `dtype` are assumed to contain missing values, and are converted to floats. Default is False.",+                    ),+                    'storage_options': Field(+                        Permissive(),+                        is_required=False,+                        description="Extra options that make sense for a particular storage connection, e.g. host, port, username, password, etc.",+                    ),+                    'include_path_column': Field(+                        Any,+                        is_required=False,+                        description="bool or str, Whether or not to include the path to each particular file. If True a new column is added to the dataframe called path. If str, sets new column name. Default is False.",+                    ),+                }+            ),+            'parquet': Permissive(+                {+                    'path': Field(+                        Any,+                        is_required=True,+                        description="str or list, Source directory for data, or path(s) to individual parquet files. Prefix with a protocol like s3:// to read from alternative filesystems. To read from multiple files you can pass a globstring or a list of paths, with the caveat that they must all have the same protocol.",+                    ),+                    'columns': Field(+                        Any,+                        is_required=False,+                        description="str or list or None (default), Field name(s) to read in as columns in the output. By default all non-index fields will be read (as determined by the pandas parquet metadata, if present). Provide a single field name instead of a list to read in the data as a Series.",+                    ),+                    'filters': Field(

+import dask.dataframe as dd++from dagster import (+    Any,+    AssetMaterialization,+    Bool,+    DagsterInvariantViolationError,+    DagsterType,+    Enum,+    EnumValue,+    EventMetadataEntry,+    Field,+    Int,+    Permissive,+    String,+    TypeCheck,+    check,+    dagster_type_loader,+    dagster_type_materializer,+)+from dagster.config.field_utils import Selector++WriteCompressionTextOptions = Enum(+    'WriteCompressionText', [EnumValue('gzip'), EnumValue('bz2'), EnumValue('xz'),],+)++EngineParquetOptions = Enum(+    'EngineParquet', [EnumValue('auto'), EnumValue('fastparquet'), EnumValue('pyarrow'),],+)+++def dict_without_keys(ddict, *keys):+    return {key: value for key, value in ddict.items() if key not in set(keys)}+++@dagster_type_materializer(+    Selector(+        {+            'csv': Permissive(+                {+                    'path': Field(+                        Any,+                        is_required=True,+                        description="str or list, Path glob indicating the naming scheme for the output files",+                    ),+                    'single_file': Field(+                        Bool,+                        is_required=False,+                        description="Whether to save everything into a single CSV file. Under the single file mode, each partition is appended at the end of the specified CSV file. Note that not all filesystems support the append mode and thus the single file mode, especially on cloud storage systems such as S3 or GCS. A warning will be issued when writing to a file that is not backed by a local filesystem.",+                    ),+                    'encoding': Field(+                        String,+                        is_required=False,+                        description="A string representing the encoding to use in the output file, defaults to 'ascii' on Python 2 and 'utf-8' on Python 3.",+                    ),+                    'mode': Field(+                        String, is_required=False, description="Python write mode, default 'w'",+                    ),+                    'compression': Field(+                        WriteCompressionTextOptions,+                        is_required=False,+                        description="a string representing the compression to use in the output file, allowed values are 'gzip', 'bz2', 'xz'",+                    ),+                    'compute': Field(+                        Bool,+                        is_required=False,+                        description="If true, immediately executes. If False, returns a set of delayed objects, which can be computed at a later time.",+                    ),+                    'storage_options': Field(+                        Permissive(),+                        is_required=False,+                        description="Parameters passed on to the backend filesystem class.",+                    ),+                    'header_first_partition_only': Field(+                        Bool,+                        is_required=False,+                        description="If set to `True`, only write the header row in the first output file. By default, headers are written to all partitions under the multiple file mode (`single_file` is `False`) and written only once under the single file mode (`single_file` is `True`). It must not be `False` under the single file mode.",+                    ),+                    'compute_kwargs': Field(+                        Permissive(),+                        is_required=False,+                        description="Options to be passed in to the compute method",+                    ),+                }+            ),+            'parquet': Permissive(+                {+                    'path': Field(+                        Any,+                        is_required=True,+                        description="str or pathlib.Path, Destination directory for data. Prepend with protocol like ``s3://`` or ``hdfs://`` for remote data.",+                    ),+                    'engine': Field(+                        EngineParquetOptions,+                        is_required=False,+                        description="{'auto', 'fastparquet', 'pyarrow'}, default 'auto' Parquet library to use. If only one library is installed, it will use that one; if both, it will use 'fastparquet'.",+                    ),+                    'compression': Field(+                        Any,+                        is_required=False,+                        description="str or dict, optional Either a string like ``'snappy'`` or a dictionary mapping column names to compressors like ``{'name': 'gzip', 'values': 'snappy'}``. The default is ``'default'``, which uses the default compression for whichever engine is selected.",+                    ),+                    'write_index': Field(+                        Bool,+                        is_required=False,+                        description="Whether or not to write the index. Defaults to True.",+                    ),+                    'append': Field(+                        Bool,+                        is_required=False,+                        description="If False (default), construct data-set from scratch. If True, add new row-group(s) to an existing data-set. In the latter case, the data-set must exist, and the schema must match the input data.",+                    ),+                    'ignore_divisions': Field(+                        Bool,+                        is_required=False,+                        description="If False (default) raises error when previous divisions overlap with the new appended divisions. Ignored if append=False.",+                    ),+                    'partition_on': Field(+                        list,+                        is_required=False,+                        description="Construct directory-based partitioning by splitting on these fields values. Each dask partition will result in one or more datafiles, there will be no global groupby.",+                    ),+                    'storage_options': Field(+                        Permissive(),+                        is_required=False,+                        description="Key/value pairs to be passed on to the file-system backend, if any.",+                    ),+                    'write_metadata_file': Field(+                        Bool,+                        is_required=False,+                        description="Whether to write the special '_metadata' file.",+                    ),+                    'compute': Field(+                        Bool,+                        is_required=False,+                        description="If True (default) then the result is computed immediately. If False then a ``dask.delayed`` object is returned for future computation.",+                    ),+                    'compute_kwargs': Field(+                        Permissive(),+                        is_required=False,+                        description="Options to be passed in to the compute method.",+                    ),+                }+            ),+            'hdf': Permissive(+                {+                    'path': Field(+                        Any,+                        is_required=True,+                        description="str or pathlib.Path, Path to a target filename. Supports strings, ``pathlib.Path``, or any object implementing the ``__fspath__`` protocol. May contain a ``*`` to denote many filenames.",+                    ),+                    'key': Field(+                        String,+                        is_required=True,+                        description="Datapath within the files. May contain a ``*`` to denote many locations",+                    ),+                    'compute': Field(+                        Bool,+                        is_required=False,+                        description="Whether or not to execute immediately.  If False then this returns a ``dask.Delayed`` value.",+                    ),+                    'scheduler': Field(+                        String,+                        is_required=False,+                        description="The scheduler to use, like 'threads' or 'processes'",+                    ),+                }+            ),+            'json': Permissive(+                {+                    'path': Field(+                        Any,+                        is_required=True,+                        description="str or list, Location to write to. If a string, and there are more than one partitions in df, should include a glob character to expand into a set of file names, or provide a ``name_function=`` parameter. Supports protocol specifications such as ``'s3://'``.",+                    ),+                    'encoding': Field(+                        String,+                        is_required=False,+                        description="default is 'utf-8', The text encoding to implement, e.g., 'utf-8'",+                    ),+                    'errors': Field(+                        String,+                        is_required=False,+                        description="default is 'strict', how to respond to errors in the conversion (see ``str.encode()``)",+                    ),+                    'storage_options': Field(+                        Permissive(),+                        is_required=False,+                        description="Passed to backend file-system implementation",+                    ),+                    'compute': Field(+                        Bool,+                        is_required=False,+                        description="If true, immediately executes. If False, returns a set of delayed objects, which can be computed at a later time.",+                    ),+                    'compute_kwargs': Field(+                        Permissive(),+                        is_required=False,+                        description="Options to be passed in to the compute method",+                    ),+                    'compression': Field(+                        String, is_required=False, description="String like 'gzip' or 'xz'.",+                    ),+                },+            ),+            'sql': Permissive(+                {+                    'name': Field(String, is_required=True, description="Name of SQL table",),+                    'uri': Field(+                        String,+                        is_required=True,+                        description="Full sqlalchemy URI for the database connection",+                    ),+                    'schema': Field(+                        String,+                        is_required=False,+                        description="Specify the schema (if database flavor supports this). If None, use default schema.",+                    ),+                    'if_exists': Field(+                        String,+                        is_required=False,+                        description="""+                            {'fail', 'replace', 'append'}, default 'fail'"+                            How to behave if the table already exists.+                            * fail: Raise a ValueError.+                            * replace: Drop the table before inserting new values.+                            * append: Insert new values to the existing table.+                        """,+                    ),+                    'index': Field(+                        Bool,+                        is_required=False,+                        description="default is True, Write DataFrame index as a column. Uses `index_label` as the column name in the table.",+                    ),+                    'index_label': Field(+                        Any,+                        is_required=False,+                        description="str or sequence, default None Column label for index column(s). If None is given (default) and `index` is True, then the index names are used. A sequence should be given if the DataFrame uses MultiIndex.",+                    ),+                    'chunksize': Field(+                        Int,+                        is_required=False,+                        description="Specify the number of rows in each batch to be written at a time. By default, all rows will be written at once",+                    ),+                    'dtype': Field(+                        Any,+                        is_required=False,+                        description="dict or scalar, Specifying the datatype for columns. If a dictionary is used, the keys should be the column names and the values should be the SQLAlchemy types or strings for the sqlite3 legacy mode. If a scalar is provided, it will be applied to all columns.",+                    ),+                    'method': Field(+                        String,+                        is_required=False,+                        description="""+                            {None, 'multi', callable}, default None+                            Controls the SQL insertion clause used:+                            * None : Uses standard SQL ``INSERT`` clause (one per row).+                            * 'multi': Pass multiple values in a single ``INSERT`` clause.+                            * callable with signature ``(pd_table, conn, keys, data_iter)``.+                            Details and a sample callable implementation can be found in the+                            section :ref:`insert method <io.sql.method>`.+                        """,+                    ),+                    'compute': Field(+                        Bool,+                        is_required=False,+                        description="default is True, When true, call dask.compute and perform the load into SQL; otherwise, return a Dask object (or array of per-block objects when parallel=True)",+                    ),+                    'parallel': Field(+                        Bool,+                        is_required=False,+                        description="default is False, When true, have each block append itself to the DB table concurrently. This can result in DB rows being in a different order than the source DataFrame's corresponding rows. When false, load each block into the SQL DB in sequence.",+                    ),+                },+            ),+        },+    )+)+def dataframe_materializer(_context, config, dask_df):+    check.inst_param(dask_df, 'dask_df', dd.DataFrame)+    file_type, file_options = list(config.items())[0]+    path = file_options.get('path')++    if file_type == 'csv':+        dask_df.to_csv(path, **dict_without_keys(file_options, 'path'))+    elif file_type == 'parquet':+        dask_df.to_parquet(path, **dict_without_keys(file_options, 'path'))+    elif file_type == 'hdf':+        dask_df.to_hdf(path, **dict_without_keys(file_options, 'path'))+    elif file_type == 'json':+        dask_df.to_json(path, **dict_without_keys(file_options, 'path'))+    elif file_type == 'sql':+        dask_df.to_sql(**file_options)+    else:+        check.failed('Unsupported file_type {file_type}'.format(file_type=file_type))++    return AssetMaterialization.file(path)+++@dagster_type_loader(+    Selector(+        {+            'csv': Permissive(+                {+                    'path': Field(+                        Any,+                        is_required=True,+                        description="str or list, Absolute or relative filepath(s). Prefix with a protocol like `s3://` to read from alternative filesystems. To read from multiple files you can pass a globstring or a list of paths, with the caveat that they must all have the same protocol.",

 commands =   !windows: /bin/bash -c '! pip list --exclude-editable | grep -e dagster -e dagit'   coverage erase   echo -e "--- \033[0;32m:pytest: Running tox tests\033[0m"+  python -m pip install "dask[dataframe]" --upgrade+  pip install pyarrow