hi
I want to configure Cortex to use SeaweedFS as the storage backend for blocks, alertmanager, and ruler data.
values.yaml for cortex
config:
blocks_storage:
backend: s3
s3:
bucket_name: cortex-blocks
endpoint: seaweedfs-s3.cortex.svc.cluster.local:8333
region: local
access_key_id: "any"
secret_access_key: "any"
insecure: true
tsdb:
dir: /data/tsdb
bucket_store:
sync_dir: /data/tsdb
bucket_index:
enabled: true
ruler_storage:
s3:
bucket_name: cortex-ruler
endpoint: seaweedfs-s3.cortex.svc.cluster.local:8333
region: local
access_key_id: "any"
secret_access_key: "any"
insecure: true
alertmanager_storage:
s3:
bucket_name: cortex-alertmanager
endpoint: seaweedfs-s3.cortex.svc.cluster.local:8333
region: local
access_key_id: "any"
secret_access_key: "any"
insecure: true
and for values.yaml for seaweedfs
filer:
s3:
enabled: true
enableAuth: true
existingConfigSecret: my-s3-secret
the buckets are created in seaweed with commands:
curl --aws-sigv4 "aws:amz:local:seaweedfs" --user "any:any" -X PUT http://localhost:8333/cortex-blocks
curl --aws-sigv4 "aws:amz:local:seaweedfs" --user "any:any" -X PUT http://localhost:8333/cortex-ruler
curl --aws-sigv4 "aws:amz:local:seaweedfs" --user "any:any" -X PUT http://localhost:8333/cortex-alertmanager

and my-s3-secret is:
apiVersion: v1
kind: Secret
type: Opaque
metadata:
name: my-s3-secret
namespace: cortex
labels:
app.kubernetes.io/name: seaweedfs
app.kubernetes.io/component: s3
stringData:
seaweedfs_s3_config: '{"identities":[{"name":"cortex","credentials":[{"accessKey":"any","secretKey":"any"}],"actions":["Admin","Read","Write"]}]}'
But i got this error:
ts=2024-10-17T14:29:38.217640249Z caller=main.go:199 level=info msg="Starting Cortex" version="(version=1.17.1, branch=HEAD, revision=62b2513)"
ts=2024-10-17T14:29:38.217898962Z caller=server.go:319 level=info http=:8080 grpc=:9095 msg="server listening on addresses"
ts=2024-10-17T14:29:38.219384879Z caller=inmemory_index_cache.go:79 level=info msg="created in-memory index cache" maxItemSizeBytes=134217728 maxSizeBytes=1073741824
ts=2024-10-17T14:29:38.226669035Z caller=module_service.go:64 level=info msg=initialising module=server
ts=2024-10-17T14:29:38.226832423Z caller=module_service.go:64 level=info msg=initialising module=runtime-config
ts=2024-10-17T14:29:38.227123335Z caller=module_service.go:64 level=info msg=initialising module=memberlist-kv
ts=2024-10-17T14:29:38.227139267Z caller=module_service.go:64 level=info msg=initialising module=store-gateway
ts=2024-10-17T14:29:38.227148886Z caller=bucket_stores.go:156 level=info msg="synchronizing TSDB blocks for all users"
ts=2024-10-17T14:31:30.344454312Z caller=bucket_client.go:136 level=error msg="bucket operation fail after retries" err="The specified bucket does not exist" operation="Iter "
ts=2024-10-17T14:33:11.156805671Z caller=bucket_client.go:136 level=error msg="bucket operation fail after retries" err="The specified bucket does not exist" operation="Iter "
ts=2024-10-17T14:34:54.148962512Z caller=bucket_client.go:136 level=error msg="bucket operation fail after retries" err="The specified bucket does not exist" operation="Iter "
ts=2024-10-17T14:34:54.149003656Z caller=bucket_stores.go:161 level=warn msg="failed to synchronize TSDB blocks" err="The specified bucket does not exist"
ts=2024-10-17T14:34:54.150152469Z caller=cortex.go:446 level=error msg="module failed" module=store-gateway err="invalid service state: Failed, expected: Running, failure: initial blocks synchronization: The specified bucket does not exist"
ts=2024-10-17T14:34:54.150194049Z caller=module_service.go:96 level=info msg="module stopped" module=runtime-config
ts=2024-10-17T14:34:54.150167487Z caller=module_service.go:96 level=info msg="module stopped" module=memberlist-kv
ts=2024-10-17T14:34:54.152305275Z caller=server_service.go:50 level=info msg="server stopped"
ts=2024-10-17T14:34:54.152323934Z caller=module_service.go:96 level=info msg="module stopped" module=server
ts=2024-10-17T14:34:54.152334791Z caller=cortex.go:435 level=info msg="Cortex stopped"
ts=2024-10-17T14:34:54.152639278Z caller=log.go:121 level=error msg="error running cortex" err="failed services\ngithub.com/cortexproject/cortex/pkg/cortex.(*Cortex).Run\n\t/__w/cortex/cortex/pkg/cortex/cortex.go:481\nmain.main\n\t/__w/cortex/cortex/cmd/cortex/main.go:201\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:267\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1650"
Stream closed EOF for cortex/cortex-store-gateway-0 (store-gateway)

within cortex-store-gateway-0 pod: there is no /etc/cortex/config.yaml but there is /etc/cortex/cortex.yaml. I got results
cat /etc/cortex/cortex.yaml
alertmanager:
cluster:
listen_address: 0.0.0.0:9094
enable_api: false
external_url: /api/prom/alertmanager
alertmanager_storage:
s3:
access_key_id: any
bucket_name: cortex-alertmanager
endpoint: seaweedfs-s3.cortex.svc.cluster.local:8333
insecure: true
region: local
secret_access_key: any
api:
prometheus_http_prefix: /prometheus
response_compression_enabled: true
auth_enabled: false
blocks_storage:
backend: s3
bucket_store:
bucket_index:
enabled: true
sync_dir: /data/tsdb
s3:
access_key_id: any
bucket_name: cortex-blocks
endpoint: seaweedfs-s3.cortex.svc.cluster.local:8333
insecure: true
region: local
secret_access_key: any
tsdb:
dir: /data/tsdb
distributor:
pool:
health_check_ingesters: true
shard_by_all_labels: true
frontend:
log_queries_longer_than: 10s
ingester:
lifecycler:
final_sleep: 30s
join_after: 10s
num_tokens: 512
observe_period: 10s
ring:
kvstore:
store: memberlist
replication_factor: 3
ingester_client:
grpc_client_config:
max_recv_msg_size: 10485760
max_send_msg_size: 10485760
limits:
enforce_metric_name: true
max_query_lookback: 0s
reject_old_samples: true
reject_old_samples_max_age: 168h
memberlist:
bind_port: 7946
join_members:

    'cortex-memberlist'
    querier:
    active_query_tracker_dir: /data/active-query-tracker
    store_gateway_addresses: |-
    dns+cortex-store-gateway-headless:9095
    query_range:
    align_queries_with_step: true
    cache_results: true
    results_cache:
    cache:
    memcached:
    expiration: 1h
    memcached_client:
    timeout: 1s

split_queries_by_interval: 24h
ruler:
enable_alertmanager_discovery: false
enable_api: true
ruler_storage:
s3:
access_key_id: any
bucket_name: cortex-ruler
endpoint: seaweedfs-s3.cortex.svc.cluster.local:8333
insecure: true
region: local
secret_access_key: any
runtime_config:
file: /etc/cortex-runtime-config/runtime_config.yaml
server:
grpc_listen_port: 9095
grpc_server_max_concurrent_streams: 10000
grpc_server_max_recv_msg_size: 10485760
grpc_server_max_send_msg_size: 10485760
http_listen_port: 8080
store_gateway:
sharding_enabled: false/
Can you tell me if the provided YAML file already has the necessary configurations, and if not, what needs to be added?