Package: simpletasks-data
simpletasks-dataAdditional tasks forsimpletasksto handle data.Provides anImportTaskto import data into aFlask-SQLAlchemymodel, from any source of data.Data sources provided are:CSV (ImportCsv)SQLAlchemy query (ImportTable)
Custom data sources can easily be implemented via inheritingImportSource.Other data sources are provided by other libraries:gapi-helperprovides Google Sheets as source.Sample:importcontextlibfromtypingimportIterable,Iterator,List,Optional,SequenceimportclickfromsimpletasksimportCli,CliParamsfromsimpletasks_dataimportImportSource,ImportTask,Mappingfrommyappimportdb@click.group()defcli():passclassAsset(db.Model):"""Model to import to"""id=db.Column(db.Integer,primary_key=True)serialnumber=db.Column(db.String(128),index=True)warehouse=db.Column(db.String(128))status=db.Column(db.String(128))product=db.Column(db.String(128))guid=db.Column(db.String(36))classAssetHistory(db.Model):"""Model to keep track of changes"""id=db.Column(db.Integer,primary_key=True)date=db.Column(db.DateTime)asset_id=db.Column(db.Integer,db.ForeignKey("asset.id"),nullable=False,index=True)asset=db.relationship("Asset",foreign_keys=asset_id)old_warehouse=db.Column(db.String(128))new_warehouse=db.Column(db.String(128))old_status=db.Column(db.String(128))new_status=db.Column(db.String(128))@Cli(cli,params=[CliParams.progress(),CliParams.dryrun()])classImportAssetsTask(ImportTask):class_AssetsSource(ImportSource):class_AssetMapping(Mapping):def__init__(self)->None:super().__init__()# Defines mapping between the input data and the fields from the model# self.<name of the field in the model> = self.auto() -- in the order of the input dataself.serialnumber=self.auto()self.status=self.auto(keep_history=True)self.warehouse=self.auto(keep_history=True)self.product=self.auto()self.guid=self.auto()# If there are gaps in the input data (i.e. fields not being used in the model), you can either:# - use `self.foobar = self.col()` instead of `self.foobar = self.auto()` to specify the column name after the gap# - use `foobar = self.auto()` to still register the gap/column, but not use it in the modeldefget_key_column_name(self)->str:# By default, we use the "id" field - this overrides itreturn"serialnumber"defget_header_line_number(self)->int:# By default we skip the first (0-index) line (header) - setting to -1 includes all linesreturn-1@contextlib.contextmanagerdefgetGeneratorData(self)->Iterator[Iterable[Sequence[str]]]:# Custom data generatoroutput:List[Sequence[str]]=[]forxino:output.append([serialnumber,status,warehouse,product,guid])yieldoutputdef__init__(self)->None:super().__init__(self._AssetMapping())defcreateModel(self)->Asset:returnAsset()defcreateHistoryModel(self,base:Asset)->Optional[AssetHistory]:o=AssetHistory()o.asset_id=base.idreturnodef__init__(self,*args,**kwargs):super().__init__(model=Asset(),keep_history=True,*args,**kwargs)defget_sources(self)->Iterable[ImportSource]:# Here we can have multiple sources if we wishreturn[self._AssetsSource()]ContributingTo initialize the environment:poetry install --no-root
poetry install -E geoalchemyTo run tests (including linting and code formatting checks), please run:poetry run pytest --mypy --flake8 && poetry run black --check .