Soutrik commited on
Commit
ff35886
·
1 Parent(s): 0ca9ca4

dvc added with s3

Browse files
Files changed (5) hide show
  1. .dvc/config +3 -3
  2. .gitignore +1 -0
  3. basic_setup.md +5 -0
  4. data.dvc +6 -0
  5. ec2_runner_setup.md +9 -1
.dvc/config CHANGED
@@ -1,7 +1,7 @@
1
  [core]
2
  autostage = true
3
  remote = aws_remote
4
- ['remote "myremote"']
5
- url = /tmp/dvcstore
6
  ['remote "aws_remote"']
7
- url = s3://deep-bucket-s3
 
1
  [core]
2
  autostage = true
3
  remote = aws_remote
4
+ ['remote "local_remote"']
5
+ url = /tmp/dvclocalstore
6
  ['remote "aws_remote"']
7
+ url = s3://mybucket/data
.gitignore CHANGED
@@ -20,3 +20,4 @@ app/core/__pycache__/
20
  src/__pycache__/test_infra.cpython-310.pyc
21
  app/core/__pycache__/config.cpython-310.pyc
22
  data/
 
 
20
  src/__pycache__/test_infra.cpython-310.pyc
21
  app/core/__pycache__/config.cpython-310.pyc
22
  data/
23
+ /data
basic_setup.md CHANGED
@@ -338,6 +338,11 @@ dvc push -r azblob
338
  Next we will add S3 as a remote
339
 
340
  ```bash
 
 
 
 
 
341
  ```
342
 
343
  9. ## __HYDRA SETUP__
 
338
  Next we will add S3 as a remote
339
 
340
  ```bash
341
+ dvc remote add --default aws_remote s3://mybucket
342
+ dvc remote modify --local aws_remote access_key_id <>
343
+ dvc remote modify --local aws_remote secret_access_key <>
344
+ dvc remote modify --local aws_remote region ap-south-1
345
+ dvc push -r aws_remote
346
  ```
347
 
348
  9. ## __HYDRA SETUP__
data.dvc ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ outs:
2
+ - md5: a372d6faac374b9f988d530864d0d7d5.dir
3
+ size: 97446370
4
+ nfiles: 3002
5
+ hash: md5
6
+ path: data
ec2_runner_setup.md CHANGED
@@ -42,4 +42,12 @@ sudo apt install unzip
42
  unzip awscliv2.zip
43
  sudo ./aws/install
44
  aws --version
45
- ```
 
 
 
 
 
 
 
 
 
42
  unzip awscliv2.zip
43
  sudo ./aws/install
44
  aws --version
45
+ aws configure
46
+
47
+ ```
48
+ ** S3 bucket operations
49
+ ```bash
50
+ aws s3 cp data s3://deep-bucket-s3/data --recursive
51
+ aws s3 ls s3://deep-bucket-s3
52
+ aws s3 rm s3://deep-bucket-s3/data --recursive
53
+ ```