Repository: microsoft/qlib
File: qlib/data/data.py (Python)

Code before:

    Provide instrument data from local data source.
    """

    def __init__(self):
        pass

    @property
    def _uri_inst(self):
        """Instrument file uri."""
        return os.path.join(C.get_data_path(), "instruments", "{}.txt")

    def _load_instruments(self, market):
        fname = self._uri_inst.format(market)
        if not os.path.exists(fname):
            raise ValueError("instruments not exists for market " + market)
        _instruments = dict()
        with open(fname) as f:
            for line in f:
                inst_time = line.strip().split()
                inst = inst_time[0]
                if len(inst_time) == 3:
                    # `day`
                    begin = inst_time[1]
                    end = inst_time[2]
                elif len(inst_time) == 5:
                    # `1min`
                    begin = inst_time[1] + " " + inst_time[2]
                    end = inst_time[3] + " " + inst_time[4]
                _instruments.setdefault(inst, []).append((pd.Timestamp(begin), pd.Timestamp(end)))
        return _instruments

    def list_instruments(self, instruments, start_time=None, end_time=None, freq="day", as_list=False):
        market = instruments["market"]
        if market in H["i"]:
            _instruments = H["i"][market]
        else:
            _instruments = self._load_instruments(market)
            H["i"][market] = _instruments
        # strip
        # use calendar boundary
        cal = Cal.calendar(freq=freq)
        start_time = pd.Timestamp(start_time or cal[0])
        end_time = pd.Timestamp(end_time or cal[-1])
        _instruments_filtered = {
            inst: list(
                filter(
                    lambda x: x[0] <= x[1],
                    [(max(start_time, x[0]), min(end_time, x[1])) for x in spans],
                )
            )

Review comment (suggestion):
This is very strange to use different logic for processing  different frequency data.
Why not using  code like  `pd.read_csv(fname, sep="\t", names=["inst", "start_datetime", "end_datetime"])`

Code after:
    Provide instrument data from local data source.
    """

    def __init__(self):
        pass

    @property
    def _uri_inst(self):
        """Instrument file uri."""
        return os.path.join(C.get_data_path(), "instruments", "{}.txt")

    def _load_instruments(self, market):
        fname = self._uri_inst.format(market)
        if not os.path.exists(fname):
            raise ValueError("instruments not exists for market " + market)

        _instruments = dict()
        df = pd.read_csv(
            fname,
            sep="\t",
            usecols=[0, 1, 2],
            names=["inst", "start_datetime", "end_datetime"],
            dtype={"inst": str},
            parse_dates=["start_datetime", "end_datetime"],
        )
        for row in df.itertuples(index=False):
            _instruments.setdefault(row[0], []).append((row[1], row[2]))
        return _instruments

    def list_instruments(self, instruments, start_time=None, end_time=None, freq="day", as_list=False):
        market = instruments["market"]
        if market in H["i"]:
            _instruments = H["i"][market]
        else:
            _instruments = self._load_instruments(market)
            H["i"][market] = _instruments
        # strip
        # use calendar boundary
        cal = Cal.calendar(freq=freq)
        start_time = pd.Timestamp(start_time or cal[0])
        end_time = pd.Timestamp(end_time or cal[-1])
        _instruments_filtered = {
            inst: list(
                filter(
                    lambda x: x[0] <= x[1],
                    [(max(start_time, x[0]), min(end_time, x[1])) for x in spans],
                )
            )
            for inst, spans in _instruments.items()
        }
        _instruments_filtered = {key: value for key, value in _instruments_filtered.items() if value}