Niv Sardi commited on
Commit
8aad435
1 Parent(s): 1a24a58

add dockerfiles

Browse files

Signed-off-by: Niv Sardi <xaiki@evilgiggle.com>

Files changed (5) hide show
  1. Dockerfile.famaf +14 -0
  2. deno/index.ts +40 -77
  3. deno/logos.ts +58 -0
  4. deno/process.ts +34 -0
  5. poormans-nc.sh +10 -0
Dockerfile.famaf ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ FROM docker.io/nvidia/cuda:11.4.3-runtime-ubuntu20.04
2
+ MAINTAINER Niv Sardi <x@filtra.me>
3
+ WORKDIR /app
4
+
5
+ RUN apt update && \
6
+ export DEBIAN_FRONTEND=noninteractive && \
7
+ apt install -y libcairo2 libglib2.0-0 libgl1 python3-pip && \
8
+ rm -rf /var/cache/apt
9
+ COPY ./yolov5/requirements.txt ./requirements.txt
10
+ RUN pip install -r ./requirements.txt --extra-index-url https://download.pytorch.org/whl/lts/1.8/cu111
11
+ RUN pip3 install torch==1.8.2 torchvision==0.9.2 torchaudio==0.8.2 --extra-index-url https://download.pytorch.org/whl/lts/1.8/cu111
12
+
13
+ CMD sh -c 'while true; do echo "🐍 waiting" ; sleep 2h; done'
14
+
deno/index.ts CHANGED
@@ -4,65 +4,9 @@ import { Application, Router } from "https://deno.land/x/oak@v9.0.0/mod.ts";
4
  import * as CSV from './csv.ts';
5
  import Puppet from './puppet.ts';
6
  import selectors from './selectors.ts';
 
7
 
8
  const puppet = new Puppet();
9
- async function get_logos(page, selector): {}[] {
10
- const logos = await page.$$(selector) || [];
11
- for (const i in logos) {
12
- const bb = await page.evaluate(e => {
13
- const { x, y, width, height } = e.getBoundingClientRect();
14
- return {
15
- x, y, width, height, top: window.screen.top, left: window.screen.left
16
- }
17
- }, logos[i])
18
- logos[i].box = bb;
19
- }
20
- return logos;
21
- }
22
-
23
- async function fetch_logos(page, id, dest) {
24
- console.error(`getting logos for: ${id}`)
25
- try {
26
- const imgs = await get_logos(page, selectors.img_logo);
27
- const ids = await get_logos(page, selectors.id_logo);
28
- const cls = await get_logos(page, selectors.class_logo);
29
- const logos = [
30
- ...imgs, ...ids, ...cls
31
- ]
32
-
33
- let annotations = '';
34
- for (const i in logos) {
35
- const bb = logos[i].box
36
- if (!bb
37
- || (bb.width < 10)
38
- || (bb.height < 10)
39
- || (bb.x + bb.width < 0)
40
- || (bb.y + bb.height < 0)) continue;
41
- console.log('got bb', bb)
42
-
43
- try {
44
- await logos[i].screenshot({
45
- path: dest
46
- .replace('images', 'logos')
47
- .replace('.png', `.${i}.png`)
48
- })
49
- annotations +=
50
- `${id} ${bb.x + bb.width / 2} ${bb.y + bb.height / 2} ${bb.width} ${bb.height}\n`
51
- } catch (e) {
52
- console.error(`couldn't screenshot logo: ${e}`);
53
- }
54
- }
55
- if (logos.length) {
56
- await Deno.writeTextFile(dest
57
- .replace('images', 'labels')
58
- .replace('png', 'txt'),
59
- annotations);
60
- }
61
- } catch (err) {
62
- console.error(`error in screenshot: ${err}`);
63
- }
64
- }
65
-
66
  const app = new Application();
67
  const router = new Router();
68
 
@@ -70,33 +14,52 @@ const stats = {
70
  in_flight: 0,
71
  done: 0
72
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
73
  router.post('/screenshot', async (ctx) => {
74
- const {request, response} = ctx;
75
- const q = await request.body().value;
76
 
77
  stats.in_flight++;
78
- const ret = await puppet.run(async page => {
79
- console.error('running', q, stats)
80
- await page.goto(q.url, {waitUntil: 'networkidle2', timeout: 60000})
81
- await page.screenshot({ path: q.path, fullPage: true })
82
- if (q.logos) {
83
- await fetch_logos(page, q.id, q.logos)
84
- }
85
- console.error(`screenshot ok: ${q.path}`)
86
- return {response: 'ok'}
 
 
 
 
 
 
 
 
 
 
87
  })
88
  stats.in_flight--;
89
  stats.done++
90
- response.body = ret
91
  })
92
- router.post('/bco', async (ctx) => {
93
- const {request, response} = ctx;
94
- const q = await request.body().value;
95
- const ret = await process(q)
96
-
97
- console.error(`ret: ${ret}`)
98
- response.body = ret
99
- });
100
 
101
  app.use(router.routes())
102
  app.use(router.allowedMethods())
 
4
  import * as CSV from './csv.ts';
5
  import Puppet from './puppet.ts';
6
  import selectors from './selectors.ts';
7
+ import { fetch_logos } from './logos.ts';
8
 
9
  const puppet = new Puppet();
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
10
  const app = new Application();
11
  const router = new Router();
12
 
 
14
  in_flight: 0,
15
  done: 0
16
  }
17
+
18
+ export const DEFAULT_VIEWPORTS = [{
19
+ width: 640,
20
+ height: 480
21
+ }, {
22
+ width: 1080,
23
+ height: 800
24
+ }, {
25
+ width: 640,
26
+ height: 640
27
+ }, {
28
+ width: 600,
29
+ height: 800,
30
+ hasTouch: true,
31
+ isMobile: true
32
+ }]
33
+
34
  router.post('/screenshot', async (ctx) => {
35
+ const { request, response } = ctx;
36
+ const { url, path = './debug.png', logos = false, viewports = DEFAULT_VIEWPORTS } = await request.body().value;
37
 
38
  stats.in_flight++;
39
+ let i = 0, v = {};
40
+ viewports.map(async (v, i) => {
41
+ const ret = await puppet.run(async page => {
42
+ await page.setViewport(v)
43
+ console.error('running', url, stats, v)
44
+ try {
45
+ const npath = path.replace('.png', `.${i}.png`)
46
+
47
+ await page.goto(url, { waitUntil: 'networkidle2', timeout: 60000 })
48
+ await page.screenshot({ path: npath, fullPage: true })
49
+ if (logos) {
50
+ await fetch_logos(page, npath)
51
+ }
52
+ console.error(`screenshot ${v} / ${i} ok: ${path}`)
53
+ } catch (e) {
54
+ console.error(e)
55
+ }
56
+ return { response: 'ok' }
57
+ })
58
  })
59
  stats.in_flight--;
60
  stats.done++
61
+ response.body = { response: 'ok' }
62
  })
 
 
 
 
 
 
 
 
63
 
64
  app.use(router.routes())
65
  app.use(router.allowedMethods())
deno/logos.ts ADDED
@@ -0,0 +1,58 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import selectors from './selectors.ts';
2
+
3
+ export async function get_logos(page, selector): {}[] {
4
+ const logos = await page.$$(selector) || [];
5
+ for (const i in logos) {
6
+ const bb = await page.evaluate(e => {
7
+ const { x, y, width, height } = e.getBoundingClientRect();
8
+ return {
9
+ x, y, width, height, top: window.screen.top, left: window.screen.left
10
+ }
11
+ }, logos[i])
12
+ logos[i].box = bb;
13
+ }
14
+ return logos;
15
+ }
16
+
17
+ export async function fetch_logos(page, dest) {
18
+ console.error(`getting logos for: ${dest}`)
19
+ try {
20
+ const imgs = await get_logos(page, selectors.img_logo);
21
+ const ids = await get_logos(page, selectors.id_logo);
22
+ const cls = await get_logos(page, selectors.class_logo);
23
+ const logos = [
24
+ ...imgs, ...ids, ...cls
25
+ ]
26
+
27
+ let annotations = '';
28
+ for (const i in logos) {
29
+ const bb = logos[i].box
30
+ if (!bb
31
+ || (bb.width < 10)
32
+ || (bb.height < 10)
33
+ || (bb.x + bb.width < 0)
34
+ || (bb.y + bb.height < 0)) continue;
35
+ console.log('got bb', bb)
36
+
37
+ try {
38
+ await logos[i].screenshot({
39
+ path: dest
40
+ .replace('images', 'logos')
41
+ .replace('.png', `.${i}.png`)
42
+ })
43
+ annotations +=
44
+ `${o.id} ${bb.x + bb.width / 2} ${bb.y + bb.height / 2} ${bb.width} ${bb.height}\n`
45
+ } catch (e) {
46
+ console.error(`couldn't screenshot logo: ${e}`);
47
+ }
48
+ }
49
+ if (logos.length) {
50
+ await Deno.writeTextFile(dest
51
+ .replace('images', 'labels')
52
+ .replace('png', 'txt'),
53
+ annotations);
54
+ }
55
+ } catch (err) {
56
+ console.error(`error in screenshot: ${err}`);
57
+ }
58
+ }
deno/process.ts ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ router.post('/bco', async (ctx) => {
2
+ const { request, response } = ctx;
3
+ const q = await request.body().value;
4
+ const ret = await process(q)
5
+
6
+ console.error(`ret: ${ret}`)
7
+ response.body = ret
8
+ })
9
+
10
+ function process(o: { id: int, url: string, bco: string, name: string }): Promise<void> {
11
+ const promises: Promise<void>[] = [];
12
+
13
+ return puppet.run(async page => {
14
+ const url = o.url.replace('http:', 'https:');
15
+ promises.push(new Promise<void>((accept, _reject) => {
16
+ page.once('load', async () => {
17
+ const dest = `./data/images/${e.bco}.chrome.full.png`
18
+ await fetch_logos(page, bco.name, dest)
19
+ await page.screenshot({ path: dest, fullPage: true })
20
+ console.log(`screenshot ok for ${o.name}`);
21
+
22
+ accept()
23
+ })
24
+ }))
25
+
26
+ try {
27
+ await page.goto(url)
28
+ .catch(() => page.goto(o.url))
29
+ } catch (e) {
30
+ console.error(`got error: ${e}`);
31
+ }
32
+ await Promise.all(promises);
33
+ })
34
+ }
poormans-nc.sh ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/sh
2
+
3
+ PORT=1234
4
+ while true; do
5
+ perl -MIO::Socket::INET -ne "BEGIN{\$l=IO::Socket::INET->new(
6
+ LocalPort=>$PORT,Proto=>\"tcp\",Listen=>5,ReuseAddr=>1);
7
+ \$l=\$l->accept} die"
8
+ ps auxf | grep geckodriver | grep -v sh | grep -v grep | awk "{print $2}" | xargs -n1 kill
9
+ sleep 1
10
+ done