2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.764698934Z caller=main.go:120 msg="Starting Loki" version="(version=3.0.0, branch=HEAD, revision=b4f7181c7a)"
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.766021053Z caller=server.go:354 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9096
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.769529378Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.770105481Z caller=table_manager.go:273 index-store=tsdb-2020-10-24 msg="query readiness setup completed" duration=1.753µs distinct_users_len=0 distinct_users=
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.770135509Z caller=shipper.go:160 index-store=tsdb-2020-10-24 msg="starting index shipper in RW mode"
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.773375082Z caller=head_manager.go:308 index-store=tsdb-2020-10-24 component=tsdb-head-manager msg="loaded wals by period" groups=0
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.773453815Z caller=manager.go:86 index-store=tsdb-2020-10-24 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.773487811Z caller=head_manager.go:308 index-store=tsdb-2020-10-24 component=tsdb-head-manager msg="loaded wals by period" groups=0
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.78000404Z caller=mapper.go:47 msg="cleaning up mapped rules directory" path=/etc/loki/rules
2024-09-25 21:57:14 level=warn ts=2024-09-25T14:57:14.780107199Z caller=mapper.go:41 msg="unable to remove user directory" path=/etc/loki/rules/fake err="unlinkat /etc/loki/rules/fake/loki-rules.yml: device or resource busy"
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.781667141Z caller=worker.go:113 component=querier msg="Starting querier worker using query-scheduler and scheduler ring for addresses"
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.787363466Z caller=module_service.go:82 msg=starting module=server
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.787413143Z caller=module_service.go:82 msg=starting module=cache-generation-loader
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.787417592Z caller=module_service.go:82 msg=starting module=memberlist-kv
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.787429393Z caller=module_service.go:82 msg=starting module=query-frontend-tripperware
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.78744748Z caller=module_service.go:82 msg=starting module=ring
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.787488358Z caller=module_service.go:82 msg=starting module=query-scheduler-ring
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.787585648Z caller=ring.go:276 msg="ring doesn't exist in KV store yet"
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.787585626Z caller=ring.go:276 msg="ring doesn't exist in KV store yet"
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.787663168Z caller=module_service.go:82 msg=starting module=store
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.787663617Z caller=basic_lifecycler.go:297 msg="instance not found in the ring" instance=880bd4c1fe2e ring=scheduler
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.787676432Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.787684239Z caller=module_service.go:82 msg=starting module=ingester-querier
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.787697935Z caller=module_service.go:82 msg=starting module=compactor
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.787750697Z caller=ring.go:276 msg="ring doesn't exist in KV store yet"
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.787797446Z caller=ringmanager.go:186 msg="waiting until scheduler is JOINING in the ring"
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.787803267Z caller=module_service.go:82 msg=starting module=ruler
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.787806764Z caller=ringmanager.go:190 msg="scheduler is JOINING in the ring"
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.787818959Z caller=basic_lifecycler.go:297 msg="instance not found in the ring" instance=880bd4c1fe2e ring=compactor
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.787827536Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.787830841Z caller=ruler.go:533 msg="ruler up and running"
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.787843425Z caller=ringmanager.go:199 msg="waiting until scheduler is ACTIVE in the ring"
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.787903301Z caller=module_service.go:82 msg=starting module=ingester
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.787928009Z caller=module_service.go:82 msg=starting module=pattern-ring-client
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.787927087Z caller=ingester.go:445 component=ingester msg="recovering from checkpoint"
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.787937469Z caller=compactor.go:410 msg="waiting until compactor is JOINING in the ring"
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.787944422Z caller=compactor.go:414 msg="compactor is JOINING in the ring"
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.787984129Z caller=compactor.go:424 msg="waiting until compactor is ACTIVE in the ring"
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.7879909Z caller=recovery.go:40 component=ingester msg="no checkpoint found, treating as no-op"
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.788044496Z caller=ingester.go:461 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=130.003µs errors=false
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.78805811Z caller=module_service.go:82 msg=starting module=pattern-ingester
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.78805701Z caller=ingester.go:467 component=ingester msg="recovering from WAL"
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.788069673Z caller=lifecycler.go:614 component=pattern-ingester msg="not loading tokens from file, tokens file path is empty"
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.788097076Z caller=lifecycler.go:639 component=pattern-ingester msg="instance not found in ring, adding with no tokens" ring=pattern-ingester
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.788174936Z caller=lifecycler.go:483 component=pattern-ingester msg="auto-joining cluster after timeout" ring=pattern-ingester
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.788240244Z caller=module_service.go:82 msg=starting module=distributor
2024-09-25 21:57:14 level=error ts=2024-09-25T14:57:14.78827454Z caller=ratestore.go:109 msg="error getting ingester clients" err="empty ring"
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.788296143Z caller=ingester.go:483 component=ingester msg="WAL segment recovery finished" elapsed=383.313µs errors=false
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.788311803Z caller=ingester.go:431 component=ingester msg="closing recoverer"
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.78832065Z caller=ingester.go:439 component=ingester msg="WAL recovery finished" time=407.53µs
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.788328775Z caller=basic_lifecycler.go:297 component=distributor msg="instance not found in the ring" instance=880bd4c1fe2e ring=distributor
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.788363613Z caller=lifecycler.go:614 component=ingester msg="not loading tokens from file, tokens file path is empty"
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.788402397Z caller=wal.go:156 msg=started component=wal
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.788429641Z caller=lifecycler.go:639 component=ingester msg="instance not found in ring, adding with no tokens" ring=ingester
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.788484827Z caller=lifecycler.go:483 component=ingester msg="auto-joining cluster after timeout" ring=ingester
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.788499597Z caller=ingester.go:613 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=6.051314581s
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.788549854Z caller=ingester.go:172 component=pattern-ingester msg="sleeping for initial delay before starting periodic flushing" delay=6.886353659s
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.792151545Z caller=manager.go:163 user=fake msg="Starting rule manager..."
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.933232675Z caller=compactor.go:428 msg="compactor is ACTIVE in the ring"
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.956398835Z caller=ringmanager.go:203 msg="scheduler is ACTIVE in the ring"
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.95656684Z caller=module_service.go:82 msg=starting module=query-scheduler
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.956699567Z caller=module_service.go:82 msg=starting module=querier
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.956756757Z caller=module_service.go:82 msg=starting module=query-frontend
2024-09-25 21:57:14 level=info ts=2024-09-25T14:57:14.956828536Z caller=loki.go:503 msg="Loki started" startup_time=208.890256ms
2024-09-25 21:57:17 level=info ts=2024-09-25T14:57:17.957597495Z caller=scheduler.go:653 msg="this scheduler is in the ReplicationSet, will now accept requests."
2024-09-25 21:57:17 level=info ts=2024-09-25T14:57:17.95762025Z caller=worker.go:210 component=querier msg="adding connection" addr=127.0.0.1:9096
2024-09-25 21:57:19 level=info ts=2024-09-25T14:57:19.934324176Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
2024-09-25 21:57:19 level=info ts=2024-09-25T14:57:19.93441431Z caller=compactor.go:518 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
2024-09-25 21:57:24 level=info ts=2024-09-25T14:57:24.957424511Z caller=frontend_scheduler_worker.go:106 msg="adding connection to scheduler" addr=127.0.0.1:9096
2024-09-25 21:58:08 level=info ts=2024-09-25T14:58:08.391435838Z caller=compat.go:67 user=fake rule_name=HighLogErrorRate rule_type=alerting query="(rate({service_name=\"loki\"} | logfmt | level=\"error\"[1m]) > 1)" query_hash=4247829063 msg="evaluating rule"
2024-09-25 21:58:08 level=info ts=2024-09-25T14:58:08.392271198Z caller=engine.go:237 component=ruler evaluation_mode=local org_id=fake traceID=29abf2de7339d376 msg="executing query" type=instant query="(rate({service_name=\"loki\"} | logfmt | level=\"error\"[1m]) > 1)" query_hash=4247829063
2024-09-25 21:58:08 level=info ts=2024-09-25T14:58:08.393629816Z caller=table_manager.go:197 index-store=tsdb-2020-10-24 msg="downloading all files for table index_19991"
2024-09-25 21:58:08 ts=2024-09-25T14:58:08.395787047Z caller=spanlogger.go:109 table-name=index_19991 user=fake level=info msg="downloaded index set at query time" duration=140.312µs
2024-09-25 21:58:08 ts=2024-09-25T14:58:08.396347573Z caller=spanlogger.go:109 table-name=index_19991 user-id=fake user=fake level=info msg="downloaded index set at query time" duration=78.442µs
2024-09-25 21:58:08 level=info ts=2024-09-25T14:58:08.396775602Z caller=metrics.go:216 component=ruler evaluation_mode=local org_id=fake traceID=29abf2de7339d376 latency=fast query="(rate({service_name=\"loki\"} | logfmt | level=\"error\"[1m]) > 1)" query_hash=4247829063 query_type=metric range_type=instant length=0s start_delta=4.795237ms end_delta=4.795367ms step=0s duration=4.421354ms status=500 limit=0 returned_lines=0 throughput=2.2MB total_bytes=9.5kB total_bytes_structured_metadata=552B lines_per_second=15606 total_lines=69 post_filter_lines=1 total_entries=0 store_chunks_download_time=0s queue_time=0s splits=0 shards=0 query_referenced_structured_metadata=true pipeline_wrapper_filtered_lines=0 chunk_refs_fetch_time=4.924052ms cache_chunk_req=0 cache_chunk_hit=0 cache_chunk_bytes_stored=0 cache_chunk_bytes_fetched=0 cache_chunk_download_time=0s cache_index_req=0 cache_index_hit=0 cache_index_download_time=0s cache_stats_results_req=0 cache_stats_results_hit=0 cache_stats_results_download_time=0s cache_volume_results_req=0 cache_volume_results_hit=0 cache_volume_results_download_time=0s cache_result_req=0 cache_result_hit=0 cache_result_download_time=0s cache_result_query_length_served=0s ingester_chunk_refs=0 ingester_chunk_downloaded=0 ingester_chunk_matches=1 ingester_requests=1 ingester_chunk_head_bytes=9.5kB ingester_chunk_compressed_bytes=0B ingester_chunk_decompressed_bytes=0B ingester_post_filter_lines=1 congestion_control_latency=0s index_total_chunks=0 index_post_bloom_filter_chunks=0 index_bloom_filter_ratio=0.00 disable_pipeline_wrappers=false
2024-09-25 21:58:14 level=info ts=2024-09-25T14:58:14.76765212Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2024-09-25 21:59:08 level=info ts=2024-09-25T14:59:08.388809089Z caller=compat.go:67 user=fake rule_name=HighLogErrorRate rule_type=alerting query="(rate({service_name=\"loki\"} | logfmt | level=\"error\"[1m]) > 1)" query_hash=4247829063 msg="evaluating rule"
2024-09-25 21:59:08 level=info ts=2024-09-25T14:59:08.389949167Z caller=engine.go:237 component=ruler evaluation_mode=local org_id=fake traceID=666c674ad325e241 msg="executing query" type=instant query="(rate({service_name=\"loki\"} | logfmt | level=\"error\"[1m]) > 1)" query_hash=4247829063
2024-09-25 21:59:08 level=info ts=2024-09-25T14:59:08.39113775Z caller=metrics.go:216 component=ruler evaluation_mode=local org_id=fake traceID=666c674ad325e241 latency=fast query="(rate({service_name=\"loki\"} | logfmt | level=\"error\"[1m]) > 1)" query_hash=4247829063 query_type=metric range_type=instant length=0s start_delta=59.999157594s end_delta=59.999157724s step=0s duration=1.070824ms status=500 limit=0 returned_lines=0 throughput=8.9MB total_bytes=9.5kB total_bytes_structured_metadata=552B lines_per_second=64436 total_lines=69 post_filter_lines=1 total_entries=0 store_chunks_download_time=0s queue_time=0s splits=0 shards=0 query_referenced_structured_metadata=true pipeline_wrapper_filtered_lines=0 chunk_refs_fetch_time=223.801µs cache_chunk_req=0 cache_chunk_hit=0 cache_chunk_bytes_stored=0 cache_chunk_bytes_fetched=0 cache_chunk_download_time=0s cache_index_req=0 cache_index_hit=0 cache_index_download_time=0s cache_stats_results_req=0 cache_stats_results_hit=0 cache_stats_results_download_time=0s cache_volume_results_req=0 cache_volume_results_hit=0 cache_volume_results_download_time=0s cache_result_req=0 cache_result_hit=0 cache_result_download_time=0s cache_result_query_length_served=0s ingester_chunk_refs=0 ingester_chunk_downloaded=0 ingester_chunk_matches=1 ingester_requests=1 ingester_chunk_head_bytes=9.5kB ingester_chunk_compressed_bytes=0B ingester_chunk_decompressed_bytes=0B ingester_post_filter_lines=1 congestion_control_latency=0s index_total_chunks=0 index_post_bloom_filter_chunks=0 index_bloom_filter_ratio=0.00 disable_pipeline_wrappers=false
2024-09-25 21:59:14 level=info ts=2024-09-25T14:59:14.766245536Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
có lỗi gì không