Spaces:
Running
Running
Merge branch 'rolling' into change-document-style-with-linter-warnings
Browse files- .gitignore +1 -0
- Cargo.lock +372 -100
- Cargo.toml +17 -7
- Dockerfile +1 -1
- README.md +2 -2
- docs/installation.md +1 -1
- public/images/barricade.png +0 -0
- public/images/filter.png +0 -0
- public/static/themes/simple.css +29 -0
- public/templates/search.html +62 -30
- src/bin/websurfx.rs +14 -0
- src/cache/cacher.rs +104 -23
- src/cache/error.rs +40 -0
- src/cache/mod.rs +1 -0
- src/config/parser.rs +65 -51
- src/config/parser_models.rs +18 -1
- src/engines/duckduckgo.rs +21 -40
- src/engines/engine_models.rs +5 -6
- src/engines/searx.rs +27 -37
- src/handler/paths.rs +58 -54
- src/lib.rs +10 -2
- src/results/aggregation_models.rs +64 -28
- src/results/aggregator.rs +65 -57
- src/results/user_agent.rs +21 -17
- src/server/routes.rs +97 -26
- websurfx/config.lua +19 -1
.gitignore
CHANGED
@@ -4,3 +4,4 @@ package-lock.json
|
|
4 |
dump.rdb
|
5 |
.vscode
|
6 |
megalinter-reports/
|
|
|
|
4 |
dump.rdb
|
5 |
.vscode
|
6 |
megalinter-reports/
|
7 |
+
dhat-heap.json
|
Cargo.lock
CHANGED
@@ -9,7 +9,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
9 |
checksum = "617a8268e3537fe1d8c9ead925fca49ef6400927ee7bc26750e90ecee14ce4b8"
|
10 |
dependencies = [
|
11 |
"bitflags 1.3.2",
|
12 |
-
"bytes 1.
|
13 |
"futures-core",
|
14 |
"futures-sink",
|
15 |
"memchr",
|
@@ -46,7 +46,7 @@ dependencies = [
|
|
46 |
"actix-web",
|
47 |
"askama_escape",
|
48 |
"bitflags 1.3.2",
|
49 |
-
"bytes 1.
|
50 |
"derive_more",
|
51 |
"futures-core",
|
52 |
"http-range",
|
@@ -57,6 +57,18 @@ dependencies = [
|
|
57 |
"pin-project-lite",
|
58 |
]
|
59 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
60 |
[[package]]
|
61 |
name = "actix-http"
|
62 |
version = "3.4.0"
|
@@ -68,10 +80,10 @@ dependencies = [
|
|
68 |
"actix-service",
|
69 |
"actix-utils",
|
70 |
"ahash",
|
71 |
-
"base64 0.21.
|
72 |
"bitflags 2.4.0",
|
73 |
"brotli",
|
74 |
-
"bytes 1.
|
75 |
"bytestring",
|
76 |
"derive_more",
|
77 |
"encoding_rs",
|
@@ -103,7 +115,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
103 |
checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb"
|
104 |
dependencies = [
|
105 |
"quote 1.0.33",
|
106 |
-
"syn 2.0.
|
107 |
]
|
108 |
|
109 |
[[package]]
|
@@ -183,7 +195,7 @@ dependencies = [
|
|
183 |
"actix-utils",
|
184 |
"actix-web-codegen",
|
185 |
"ahash",
|
186 |
-
"bytes 1.
|
187 |
"bytestring",
|
188 |
"cfg-if 1.0.0",
|
189 |
"cookie 0.16.2",
|
@@ -216,7 +228,7 @@ dependencies = [
|
|
216 |
"actix-router",
|
217 |
"proc-macro2 1.0.66",
|
218 |
"quote 1.0.33",
|
219 |
-
"syn 2.0.
|
220 |
]
|
221 |
|
222 |
[[package]]
|
@@ -278,9 +290,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
|
|
278 |
|
279 |
[[package]]
|
280 |
name = "anstyle"
|
281 |
-
version = "1.0.
|
282 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
283 |
-
checksum = "
|
284 |
|
285 |
[[package]]
|
286 |
name = "anyhow"
|
@@ -288,12 +300,24 @@ version = "1.0.75"
|
|
288 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
289 |
checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6"
|
290 |
|
|
|
|
|
|
|
|
|
|
|
|
|
291 |
[[package]]
|
292 |
name = "askama_escape"
|
293 |
version = "0.10.3"
|
294 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
295 |
checksum = "619743e34b5ba4e9703bba34deac3427c72507c7159f5fd030aea8cac0cfe341"
|
296 |
|
|
|
|
|
|
|
|
|
|
|
|
|
297 |
[[package]]
|
298 |
name = "async-trait"
|
299 |
version = "0.1.73"
|
@@ -302,7 +326,7 @@ checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0"
|
|
302 |
dependencies = [
|
303 |
"proc-macro2 1.0.66",
|
304 |
"quote 1.0.33",
|
305 |
-
"syn 2.0.
|
306 |
]
|
307 |
|
308 |
[[package]]
|
@@ -346,9 +370,9 @@ dependencies = [
|
|
346 |
|
347 |
[[package]]
|
348 |
name = "base64"
|
349 |
-
version = "0.21.
|
350 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
351 |
-
checksum = "
|
352 |
|
353 |
[[package]]
|
354 |
name = "bit-set"
|
@@ -441,9 +465,9 @@ dependencies = [
|
|
441 |
|
442 |
[[package]]
|
443 |
name = "bytes"
|
444 |
-
version = "1.
|
445 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
446 |
-
checksum = "
|
447 |
|
448 |
[[package]]
|
449 |
name = "bytestring"
|
@@ -451,7 +475,7 @@ version = "1.3.0"
|
|
451 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
452 |
checksum = "238e4886760d98c4f899360c834fa93e62cf7f721ac3c2da375cbdf4b8679aae"
|
453 |
dependencies = [
|
454 |
-
"bytes 1.
|
455 |
]
|
456 |
|
457 |
[[package]]
|
@@ -558,8 +582,12 @@ version = "4.6.6"
|
|
558 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
559 |
checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4"
|
560 |
dependencies = [
|
561 |
-
"bytes 1.
|
|
|
562 |
"memchr",
|
|
|
|
|
|
|
563 |
]
|
564 |
|
565 |
[[package]]
|
@@ -574,7 +602,7 @@ version = "0.12.0"
|
|
574 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
575 |
checksum = "888604f00b3db336d2af898ec3c1d5d0ddf5e6d462220f2ededc33a87ac4bbd5"
|
576 |
dependencies = [
|
577 |
-
"time 0.1.
|
578 |
"url 1.7.2",
|
579 |
]
|
580 |
|
@@ -602,7 +630,7 @@ dependencies = [
|
|
602 |
"publicsuffix",
|
603 |
"serde",
|
604 |
"serde_json",
|
605 |
-
"time 0.1.
|
606 |
"try_from",
|
607 |
"url 1.7.2",
|
608 |
]
|
@@ -798,7 +826,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
798 |
checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331"
|
799 |
dependencies = [
|
800 |
"quote 1.0.33",
|
801 |
-
"syn 2.0.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
802 |
]
|
803 |
|
804 |
[[package]]
|
@@ -820,6 +861,22 @@ dependencies = [
|
|
820 |
"syn 1.0.109",
|
821 |
]
|
822 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
823 |
[[package]]
|
824 |
name = "digest"
|
825 |
version = "0.10.7"
|
@@ -918,9 +975,9 @@ dependencies = [
|
|
918 |
|
919 |
[[package]]
|
920 |
name = "error-stack"
|
921 |
-
version = "0.4.
|
922 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
923 |
-
checksum = "
|
924 |
dependencies = [
|
925 |
"anyhow",
|
926 |
"rustc_version 0.4.0",
|
@@ -1049,6 +1106,21 @@ version = "0.1.31"
|
|
1049 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1050 |
checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678"
|
1051 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1052 |
[[package]]
|
1053 |
name = "futures-channel"
|
1054 |
version = "0.3.28"
|
@@ -1056,6 +1128,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1056 |
checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2"
|
1057 |
dependencies = [
|
1058 |
"futures-core",
|
|
|
1059 |
]
|
1060 |
|
1061 |
[[package]]
|
@@ -1070,10 +1143,38 @@ version = "0.1.8"
|
|
1070 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1071 |
checksum = "ab90cde24b3319636588d0c35fe03b1333857621051837ed769faefb4c2162e4"
|
1072 |
dependencies = [
|
1073 |
-
"futures",
|
1074 |
"num_cpus",
|
1075 |
]
|
1076 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1077 |
[[package]]
|
1078 |
name = "futures-sink"
|
1079 |
version = "0.3.28"
|
@@ -1086,16 +1187,28 @@ version = "0.3.28"
|
|
1086 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1087 |
checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65"
|
1088 |
|
|
|
|
|
|
|
|
|
|
|
|
|
1089 |
[[package]]
|
1090 |
name = "futures-util"
|
1091 |
version = "0.3.28"
|
1092 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1093 |
checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533"
|
1094 |
dependencies = [
|
|
|
1095 |
"futures-core",
|
|
|
|
|
|
|
1096 |
"futures-task",
|
|
|
1097 |
"pin-project-lite",
|
1098 |
"pin-utils",
|
|
|
1099 |
]
|
1100 |
|
1101 |
[[package]]
|
@@ -1143,6 +1256,24 @@ version = "0.28.0"
|
|
1143 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1144 |
checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0"
|
1145 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1146 |
[[package]]
|
1147 |
name = "h2"
|
1148 |
version = "0.1.26"
|
@@ -1152,7 +1283,7 @@ dependencies = [
|
|
1152 |
"byteorder",
|
1153 |
"bytes 0.4.12",
|
1154 |
"fnv",
|
1155 |
-
"futures",
|
1156 |
"http 0.1.21",
|
1157 |
"indexmap",
|
1158 |
"log",
|
@@ -1167,7 +1298,7 @@ version = "0.3.21"
|
|
1167 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1168 |
checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833"
|
1169 |
dependencies = [
|
1170 |
-
"bytes 1.
|
1171 |
"fnv",
|
1172 |
"futures-core",
|
1173 |
"futures-sink",
|
@@ -1207,6 +1338,12 @@ version = "0.12.3"
|
|
1207 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1208 |
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
|
1209 |
|
|
|
|
|
|
|
|
|
|
|
|
|
1210 |
[[package]]
|
1211 |
name = "hermit-abi"
|
1212 |
version = "0.3.2"
|
@@ -1258,7 +1395,7 @@ version = "0.2.9"
|
|
1258 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1259 |
checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482"
|
1260 |
dependencies = [
|
1261 |
-
"bytes 1.
|
1262 |
"fnv",
|
1263 |
"itoa 1.0.9",
|
1264 |
]
|
@@ -1270,7 +1407,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1270 |
checksum = "6741c859c1b2463a423a1dbce98d418e6c3c3fc720fb0d45528657320920292d"
|
1271 |
dependencies = [
|
1272 |
"bytes 0.4.12",
|
1273 |
-
"futures",
|
1274 |
"http 0.1.21",
|
1275 |
"tokio-buf",
|
1276 |
]
|
@@ -1281,7 +1418,7 @@ version = "0.4.5"
|
|
1281 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1282 |
checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1"
|
1283 |
dependencies = [
|
1284 |
-
"bytes 1.
|
1285 |
"http 0.2.9",
|
1286 |
"pin-project-lite",
|
1287 |
]
|
@@ -1317,7 +1454,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1317 |
checksum = "5c843caf6296fc1f93444735205af9ed4e109a539005abb2564ae1d6fad34c52"
|
1318 |
dependencies = [
|
1319 |
"bytes 0.4.12",
|
1320 |
-
"futures",
|
1321 |
"futures-cpupool",
|
1322 |
"h2 0.1.26",
|
1323 |
"http 0.1.21",
|
@@ -1328,7 +1465,7 @@ dependencies = [
|
|
1328 |
"log",
|
1329 |
"net2",
|
1330 |
"rustc_version 0.2.3",
|
1331 |
-
"time 0.1.
|
1332 |
"tokio 0.1.22",
|
1333 |
"tokio-buf",
|
1334 |
"tokio-executor",
|
@@ -1346,7 +1483,7 @@ version = "0.14.27"
|
|
1346 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1347 |
checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468"
|
1348 |
dependencies = [
|
1349 |
-
"bytes 1.
|
1350 |
"futures-channel",
|
1351 |
"futures-core",
|
1352 |
"futures-util",
|
@@ -1371,7 +1508,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1371 |
checksum = "3a800d6aa50af4b5850b2b0f659625ce9504df908e9733b635720483be26174f"
|
1372 |
dependencies = [
|
1373 |
"bytes 0.4.12",
|
1374 |
-
"futures",
|
1375 |
"hyper 0.12.36",
|
1376 |
"native-tls",
|
1377 |
"tokio-io",
|
@@ -1383,7 +1520,7 @@ version = "0.5.0"
|
|
1383 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1384 |
checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
|
1385 |
dependencies = [
|
1386 |
-
"bytes 1.
|
1387 |
"hyper 0.14.27",
|
1388 |
"native-tls",
|
1389 |
"tokio 1.32.0",
|
@@ -1429,7 +1566,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1429 |
checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
|
1430 |
dependencies = [
|
1431 |
"autocfg 1.1.0",
|
1432 |
-
"hashbrown",
|
1433 |
]
|
1434 |
|
1435 |
[[package]]
|
@@ -1525,11 +1662,21 @@ version = "0.2.147"
|
|
1525 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1526 |
checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3"
|
1527 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1528 |
[[package]]
|
1529 |
name = "linux-raw-sys"
|
1530 |
-
version = "0.4.
|
1531 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1532 |
-
checksum = "
|
1533 |
|
1534 |
[[package]]
|
1535 |
name = "local-channel"
|
@@ -1580,6 +1727,15 @@ version = "0.1.1"
|
|
1580 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1581 |
checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
|
1582 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1583 |
[[package]]
|
1584 |
name = "markup5ever"
|
1585 |
version = "0.8.1"
|
@@ -1653,6 +1809,15 @@ dependencies = [
|
|
1653 |
"autocfg 1.1.0",
|
1654 |
]
|
1655 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1656 |
[[package]]
|
1657 |
name = "mime"
|
1658 |
version = "0.3.17"
|
@@ -1678,6 +1843,16 @@ dependencies = [
|
|
1678 |
"adler",
|
1679 |
]
|
1680 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1681 |
[[package]]
|
1682 |
name = "mio"
|
1683 |
version = "0.6.23"
|
@@ -1721,6 +1896,20 @@ dependencies = [
|
|
1721 |
"ws2_32-sys",
|
1722 |
]
|
1723 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1724 |
[[package]]
|
1725 |
name = "native-tls"
|
1726 |
version = "0.2.11"
|
@@ -1762,6 +1951,18 @@ version = "0.5.0"
|
|
1762 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1763 |
checksum = "ab250442c86f1850815b5d268639dff018c0627022bc1940eb2d642ca1ce12f0"
|
1764 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1765 |
[[package]]
|
1766 |
name = "num-traits"
|
1767 |
version = "0.2.16"
|
@@ -1825,7 +2026,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
|
|
1825 |
dependencies = [
|
1826 |
"proc-macro2 1.0.66",
|
1827 |
"quote 1.0.33",
|
1828 |
-
"syn 2.0.
|
1829 |
]
|
1830 |
|
1831 |
[[package]]
|
@@ -1836,9 +2037,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
|
|
1836 |
|
1837 |
[[package]]
|
1838 |
name = "openssl-sys"
|
1839 |
-
version = "0.9.
|
1840 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1841 |
-
checksum = "
|
1842 |
dependencies = [
|
1843 |
"cc",
|
1844 |
"libc",
|
@@ -1944,7 +2145,7 @@ dependencies = [
|
|
1944 |
"pest_meta",
|
1945 |
"proc-macro2 1.0.66",
|
1946 |
"quote 1.0.33",
|
1947 |
-
"syn 2.0.
|
1948 |
]
|
1949 |
|
1950 |
[[package]]
|
@@ -2046,7 +2247,7 @@ dependencies = [
|
|
2046 |
"phf_shared 0.11.2",
|
2047 |
"proc-macro2 1.0.66",
|
2048 |
"quote 1.0.33",
|
2049 |
-
"syn 2.0.
|
2050 |
]
|
2051 |
|
2052 |
[[package]]
|
@@ -2076,6 +2277,26 @@ dependencies = [
|
|
2076 |
"siphasher 0.3.11",
|
2077 |
]
|
2078 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2079 |
[[package]]
|
2080 |
name = "pin-project-lite"
|
2081 |
version = "0.2.13"
|
@@ -2162,6 +2383,22 @@ dependencies = [
|
|
2162 |
"url 2.4.1",
|
2163 |
]
|
2164 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2165 |
[[package]]
|
2166 |
name = "quote"
|
2167 |
version = "0.6.13"
|
@@ -2316,6 +2553,15 @@ dependencies = [
|
|
2316 |
"rand_core 0.3.1",
|
2317 |
]
|
2318 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2319 |
[[package]]
|
2320 |
name = "rayon"
|
2321 |
version = "1.7.0"
|
@@ -2353,12 +2599,21 @@ version = "0.23.3"
|
|
2353 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
2354 |
checksum = "4f49cdc0bb3f412bf8e7d1bd90fe1d9eb10bc5c399ba90973c14662a27b3f8ba"
|
2355 |
dependencies = [
|
|
|
|
|
|
|
2356 |
"combine",
|
|
|
|
|
2357 |
"itoa 1.0.9",
|
2358 |
"percent-encoding 2.3.0",
|
|
|
2359 |
"ryu",
|
2360 |
"sha1_smol",
|
2361 |
"socket2 0.4.9",
|
|
|
|
|
|
|
2362 |
"url 2.4.1",
|
2363 |
]
|
2364 |
|
@@ -2418,7 +2673,7 @@ dependencies = [
|
|
2418 |
"cookie_store",
|
2419 |
"encoding_rs",
|
2420 |
"flate2",
|
2421 |
-
"futures",
|
2422 |
"http 0.1.21",
|
2423 |
"hyper 0.12.36",
|
2424 |
"hyper-tls 0.3.2",
|
@@ -2429,7 +2684,7 @@ dependencies = [
|
|
2429 |
"serde",
|
2430 |
"serde_json",
|
2431 |
"serde_urlencoded 0.5.5",
|
2432 |
-
"time 0.1.
|
2433 |
"tokio 0.1.22",
|
2434 |
"tokio-executor",
|
2435 |
"tokio-io",
|
@@ -2446,8 +2701,8 @@ version = "0.11.20"
|
|
2446 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
2447 |
checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1"
|
2448 |
dependencies = [
|
2449 |
-
"base64 0.21.
|
2450 |
-
"bytes 1.
|
2451 |
"encoding_rs",
|
2452 |
"futures-core",
|
2453 |
"futures-util",
|
@@ -2477,36 +2732,18 @@ dependencies = [
|
|
2477 |
"winreg 0.50.0",
|
2478 |
]
|
2479 |
|
2480 |
-
[[package]]
|
2481 |
-
name = "rlua"
|
2482 |
-
version = "0.19.7"
|
2483 |
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
2484 |
-
checksum = "5d33e5ba15c3d43178f283ed5863d4531e292fc0e56fb773f3bea45f18e3a42a"
|
2485 |
-
dependencies = [
|
2486 |
-
"bitflags 1.3.2",
|
2487 |
-
"bstr",
|
2488 |
-
"libc",
|
2489 |
-
"num-traits",
|
2490 |
-
"rlua-lua54-sys",
|
2491 |
-
]
|
2492 |
-
|
2493 |
-
[[package]]
|
2494 |
-
name = "rlua-lua54-sys"
|
2495 |
-
version = "0.1.6"
|
2496 |
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
2497 |
-
checksum = "7aafabafe1895cb4a2be81a56d7ff3d46bf4b5d2f9cfdbea2ed404cdabe96474"
|
2498 |
-
dependencies = [
|
2499 |
-
"cc",
|
2500 |
-
"libc",
|
2501 |
-
"pkg-config",
|
2502 |
-
]
|
2503 |
-
|
2504 |
[[package]]
|
2505 |
name = "rustc-demangle"
|
2506 |
version = "0.1.23"
|
2507 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
2508 |
checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
|
2509 |
|
|
|
|
|
|
|
|
|
|
|
|
|
2510 |
[[package]]
|
2511 |
name = "rustc_version"
|
2512 |
version = "0.2.3"
|
@@ -2527,9 +2764,9 @@ dependencies = [
|
|
2527 |
|
2528 |
[[package]]
|
2529 |
name = "rustix"
|
2530 |
-
version = "0.38.
|
2531 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
2532 |
-
checksum = "
|
2533 |
dependencies = [
|
2534 |
"bitflags 2.4.0",
|
2535 |
"errno",
|
@@ -2687,14 +2924,14 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2"
|
|
2687 |
dependencies = [
|
2688 |
"proc-macro2 1.0.66",
|
2689 |
"quote 1.0.33",
|
2690 |
-
"syn 2.0.
|
2691 |
]
|
2692 |
|
2693 |
[[package]]
|
2694 |
name = "serde_json"
|
2695 |
-
version = "1.0.
|
2696 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
2697 |
-
checksum = "
|
2698 |
dependencies = [
|
2699 |
"itoa 1.0.9",
|
2700 |
"ryu",
|
@@ -2806,6 +3043,9 @@ name = "smallvec"
|
|
2806 |
version = "1.11.0"
|
2807 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
2808 |
checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9"
|
|
|
|
|
|
|
2809 |
|
2810 |
[[package]]
|
2811 |
name = "socket2"
|
@@ -2926,9 +3166,9 @@ dependencies = [
|
|
2926 |
|
2927 |
[[package]]
|
2928 |
name = "syn"
|
2929 |
-
version = "2.0.
|
2930 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
2931 |
-
checksum = "
|
2932 |
dependencies = [
|
2933 |
"proc-macro2 1.0.66",
|
2934 |
"quote 1.0.33",
|
@@ -2947,6 +3187,16 @@ dependencies = [
|
|
2947 |
"unicode-xid 0.2.4",
|
2948 |
]
|
2949 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2950 |
[[package]]
|
2951 |
name = "tempfile"
|
2952 |
version = "3.8.0"
|
@@ -2997,17 +3247,22 @@ checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35"
|
|
2997 |
dependencies = [
|
2998 |
"proc-macro2 1.0.66",
|
2999 |
"quote 1.0.33",
|
3000 |
-
"syn 2.0.
|
3001 |
]
|
3002 |
|
|
|
|
|
|
|
|
|
|
|
|
|
3003 |
[[package]]
|
3004 |
name = "time"
|
3005 |
-
version = "0.1.
|
3006 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
3007 |
-
checksum = "
|
3008 |
dependencies = [
|
3009 |
"libc",
|
3010 |
-
"wasi 0.10.0+wasi-snapshot-preview1",
|
3011 |
"winapi 0.3.9",
|
3012 |
]
|
3013 |
|
@@ -3071,7 +3326,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
3071 |
checksum = "5a09c0b5bb588872ab2f09afa13ee6e9dac11e10a0ec9e8e3ba39a5a5d530af6"
|
3072 |
dependencies = [
|
3073 |
"bytes 0.4.12",
|
3074 |
-
"futures",
|
3075 |
"mio 0.6.23",
|
3076 |
"num_cpus",
|
3077 |
"tokio-current-thread",
|
@@ -3090,7 +3345,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
3090 |
checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9"
|
3091 |
dependencies = [
|
3092 |
"backtrace",
|
3093 |
-
"bytes 1.
|
3094 |
"libc",
|
3095 |
"mio 0.8.8",
|
3096 |
"num_cpus",
|
@@ -3110,7 +3365,7 @@ checksum = "8fb220f46c53859a4b7ec083e41dec9778ff0b1851c0942b211edb89e0ccdc46"
|
|
3110 |
dependencies = [
|
3111 |
"bytes 0.4.12",
|
3112 |
"either",
|
3113 |
-
"futures",
|
3114 |
]
|
3115 |
|
3116 |
[[package]]
|
@@ -3119,7 +3374,7 @@ version = "0.1.7"
|
|
3119 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
3120 |
checksum = "b1de0e32a83f131e002238d7ccde18211c0a5397f60cbfffcb112868c2e0e20e"
|
3121 |
dependencies = [
|
3122 |
-
"futures",
|
3123 |
"tokio-executor",
|
3124 |
]
|
3125 |
|
@@ -3130,7 +3385,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
3130 |
checksum = "fb2d1b8f4548dbf5e1f7818512e9c406860678f29c300cdf0ebac72d1a3a1671"
|
3131 |
dependencies = [
|
3132 |
"crossbeam-utils 0.7.2",
|
3133 |
-
"futures",
|
3134 |
]
|
3135 |
|
3136 |
[[package]]
|
@@ -3140,7 +3395,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
3140 |
checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674"
|
3141 |
dependencies = [
|
3142 |
"bytes 0.4.12",
|
3143 |
-
"futures",
|
3144 |
"log",
|
3145 |
]
|
3146 |
|
@@ -3152,7 +3407,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
|
|
3152 |
dependencies = [
|
3153 |
"proc-macro2 1.0.66",
|
3154 |
"quote 1.0.33",
|
3155 |
-
"syn 2.0.
|
3156 |
]
|
3157 |
|
3158 |
[[package]]
|
@@ -3172,7 +3427,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
3172 |
checksum = "09bc590ec4ba8ba87652da2068d150dcada2cfa2e07faae270a5e0409aa51351"
|
3173 |
dependencies = [
|
3174 |
"crossbeam-utils 0.7.2",
|
3175 |
-
"futures",
|
3176 |
"lazy_static",
|
3177 |
"log",
|
3178 |
"mio 0.6.23",
|
@@ -3184,6 +3439,17 @@ dependencies = [
|
|
3184 |
"tokio-sync",
|
3185 |
]
|
3186 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
3187 |
[[package]]
|
3188 |
name = "tokio-sync"
|
3189 |
version = "0.1.8"
|
@@ -3191,7 +3457,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
3191 |
checksum = "edfe50152bc8164fcc456dab7891fa9bf8beaf01c5ee7e1dd43a397c3cf87dee"
|
3192 |
dependencies = [
|
3193 |
"fnv",
|
3194 |
-
"futures",
|
3195 |
]
|
3196 |
|
3197 |
[[package]]
|
@@ -3201,7 +3467,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
3201 |
checksum = "98df18ed66e3b72e742f185882a9e201892407957e45fbff8da17ae7a7c51f72"
|
3202 |
dependencies = [
|
3203 |
"bytes 0.4.12",
|
3204 |
-
"futures",
|
3205 |
"iovec",
|
3206 |
"mio 0.6.23",
|
3207 |
"tokio-io",
|
@@ -3217,7 +3483,7 @@ dependencies = [
|
|
3217 |
"crossbeam-deque 0.7.4",
|
3218 |
"crossbeam-queue",
|
3219 |
"crossbeam-utils 0.7.2",
|
3220 |
-
"futures",
|
3221 |
"lazy_static",
|
3222 |
"log",
|
3223 |
"num_cpus",
|
@@ -3232,7 +3498,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
3232 |
checksum = "93044f2d313c95ff1cb7809ce9a7a05735b012288a888b62d4434fd58c94f296"
|
3233 |
dependencies = [
|
3234 |
"crossbeam-utils 0.7.2",
|
3235 |
-
"futures",
|
3236 |
"slab",
|
3237 |
"tokio-executor",
|
3238 |
]
|
@@ -3243,7 +3509,7 @@ version = "0.7.8"
|
|
3243 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
3244 |
checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d"
|
3245 |
dependencies = [
|
3246 |
-
"bytes 1.
|
3247 |
"futures-core",
|
3248 |
"futures-sink",
|
3249 |
"pin-project-lite",
|
@@ -3413,9 +3679,9 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
|
|
3413 |
|
3414 |
[[package]]
|
3415 |
name = "walkdir"
|
3416 |
-
version = "2.
|
3417 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
3418 |
-
checksum = "
|
3419 |
dependencies = [
|
3420 |
"same-file",
|
3421 |
"winapi-util",
|
@@ -3427,7 +3693,7 @@ version = "0.2.0"
|
|
3427 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
3428 |
checksum = "b6395efa4784b027708f7451087e647ec73cc74f5d9bc2e418404248d679a230"
|
3429 |
dependencies = [
|
3430 |
-
"futures",
|
3431 |
"log",
|
3432 |
"try-lock",
|
3433 |
]
|
@@ -3443,9 +3709,9 @@ dependencies = [
|
|
3443 |
|
3444 |
[[package]]
|
3445 |
name = "wasi"
|
3446 |
-
version = "0.10.
|
3447 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
3448 |
-
checksum = "
|
3449 |
|
3450 |
[[package]]
|
3451 |
name = "wasi"
|
@@ -3474,7 +3740,7 @@ dependencies = [
|
|
3474 |
"once_cell",
|
3475 |
"proc-macro2 1.0.66",
|
3476 |
"quote 1.0.33",
|
3477 |
-
"syn 2.0.
|
3478 |
"wasm-bindgen-shared",
|
3479 |
]
|
3480 |
|
@@ -3508,7 +3774,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
|
|
3508 |
dependencies = [
|
3509 |
"proc-macro2 1.0.66",
|
3510 |
"quote 1.0.33",
|
3511 |
-
"syn 2.0.
|
3512 |
"wasm-bindgen-backend",
|
3513 |
"wasm-bindgen-shared",
|
3514 |
]
|
@@ -3535,25 +3801,31 @@ version = "0.20.1"
|
|
3535 |
dependencies = [
|
3536 |
"actix-cors",
|
3537 |
"actix-files",
|
|
|
3538 |
"actix-web",
|
|
|
3539 |
"async-trait",
|
3540 |
"criterion",
|
|
|
3541 |
"env_logger",
|
3542 |
"error-stack",
|
3543 |
"fake-useragent",
|
|
|
3544 |
"handlebars",
|
3545 |
"log",
|
3546 |
"md5",
|
|
|
|
|
3547 |
"once_cell",
|
3548 |
"rand 0.8.5",
|
3549 |
"redis",
|
3550 |
"regex",
|
3551 |
"reqwest 0.11.20",
|
3552 |
-
"rlua",
|
3553 |
"rusty-hook",
|
3554 |
"scraper",
|
3555 |
"serde",
|
3556 |
"serde_json",
|
|
|
3557 |
"tempfile",
|
3558 |
"tokio 1.32.0",
|
3559 |
]
|
|
|
9 |
checksum = "617a8268e3537fe1d8c9ead925fca49ef6400927ee7bc26750e90ecee14ce4b8"
|
10 |
dependencies = [
|
11 |
"bitflags 1.3.2",
|
12 |
+
"bytes 1.5.0",
|
13 |
"futures-core",
|
14 |
"futures-sink",
|
15 |
"memchr",
|
|
|
46 |
"actix-web",
|
47 |
"askama_escape",
|
48 |
"bitflags 1.3.2",
|
49 |
+
"bytes 1.5.0",
|
50 |
"derive_more",
|
51 |
"futures-core",
|
52 |
"http-range",
|
|
|
57 |
"pin-project-lite",
|
58 |
]
|
59 |
|
60 |
+
[[package]]
|
61 |
+
name = "actix-governor"
|
62 |
+
version = "0.4.1"
|
63 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
64 |
+
checksum = "46ff2d40f2bc627b8054c5e20fa6b0b0cf9428699b54bd41634e9ae3098ad555"
|
65 |
+
dependencies = [
|
66 |
+
"actix-http",
|
67 |
+
"actix-web",
|
68 |
+
"futures 0.3.28",
|
69 |
+
"governor",
|
70 |
+
]
|
71 |
+
|
72 |
[[package]]
|
73 |
name = "actix-http"
|
74 |
version = "3.4.0"
|
|
|
80 |
"actix-service",
|
81 |
"actix-utils",
|
82 |
"ahash",
|
83 |
+
"base64 0.21.4",
|
84 |
"bitflags 2.4.0",
|
85 |
"brotli",
|
86 |
+
"bytes 1.5.0",
|
87 |
"bytestring",
|
88 |
"derive_more",
|
89 |
"encoding_rs",
|
|
|
115 |
checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb"
|
116 |
dependencies = [
|
117 |
"quote 1.0.33",
|
118 |
+
"syn 2.0.32",
|
119 |
]
|
120 |
|
121 |
[[package]]
|
|
|
195 |
"actix-utils",
|
196 |
"actix-web-codegen",
|
197 |
"ahash",
|
198 |
+
"bytes 1.5.0",
|
199 |
"bytestring",
|
200 |
"cfg-if 1.0.0",
|
201 |
"cookie 0.16.2",
|
|
|
228 |
"actix-router",
|
229 |
"proc-macro2 1.0.66",
|
230 |
"quote 1.0.33",
|
231 |
+
"syn 2.0.32",
|
232 |
]
|
233 |
|
234 |
[[package]]
|
|
|
290 |
|
291 |
[[package]]
|
292 |
name = "anstyle"
|
293 |
+
version = "1.0.3"
|
294 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
295 |
+
checksum = "b84bf0a05bbb2a83e5eb6fa36bb6e87baa08193c35ff52bbf6b38d8af2890e46"
|
296 |
|
297 |
[[package]]
|
298 |
name = "anyhow"
|
|
|
300 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
301 |
checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6"
|
302 |
|
303 |
+
[[package]]
|
304 |
+
name = "arc-swap"
|
305 |
+
version = "1.6.0"
|
306 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
307 |
+
checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6"
|
308 |
+
|
309 |
[[package]]
|
310 |
name = "askama_escape"
|
311 |
version = "0.10.3"
|
312 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
313 |
checksum = "619743e34b5ba4e9703bba34deac3427c72507c7159f5fd030aea8cac0cfe341"
|
314 |
|
315 |
+
[[package]]
|
316 |
+
name = "async-once-cell"
|
317 |
+
version = "0.5.3"
|
318 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
319 |
+
checksum = "9338790e78aa95a416786ec8389546c4b6a1dfc3dc36071ed9518a9413a542eb"
|
320 |
+
|
321 |
[[package]]
|
322 |
name = "async-trait"
|
323 |
version = "0.1.73"
|
|
|
326 |
dependencies = [
|
327 |
"proc-macro2 1.0.66",
|
328 |
"quote 1.0.33",
|
329 |
+
"syn 2.0.32",
|
330 |
]
|
331 |
|
332 |
[[package]]
|
|
|
370 |
|
371 |
[[package]]
|
372 |
name = "base64"
|
373 |
+
version = "0.21.4"
|
374 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
375 |
+
checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2"
|
376 |
|
377 |
[[package]]
|
378 |
name = "bit-set"
|
|
|
465 |
|
466 |
[[package]]
|
467 |
name = "bytes"
|
468 |
+
version = "1.5.0"
|
469 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
470 |
+
checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223"
|
471 |
|
472 |
[[package]]
|
473 |
name = "bytestring"
|
|
|
475 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
476 |
checksum = "238e4886760d98c4f899360c834fa93e62cf7f721ac3c2da375cbdf4b8679aae"
|
477 |
dependencies = [
|
478 |
+
"bytes 1.5.0",
|
479 |
]
|
480 |
|
481 |
[[package]]
|
|
|
582 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
583 |
checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4"
|
584 |
dependencies = [
|
585 |
+
"bytes 1.5.0",
|
586 |
+
"futures-core",
|
587 |
"memchr",
|
588 |
+
"pin-project-lite",
|
589 |
+
"tokio 1.32.0",
|
590 |
+
"tokio-util",
|
591 |
]
|
592 |
|
593 |
[[package]]
|
|
|
602 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
603 |
checksum = "888604f00b3db336d2af898ec3c1d5d0ddf5e6d462220f2ededc33a87ac4bbd5"
|
604 |
dependencies = [
|
605 |
+
"time 0.1.43",
|
606 |
"url 1.7.2",
|
607 |
]
|
608 |
|
|
|
630 |
"publicsuffix",
|
631 |
"serde",
|
632 |
"serde_json",
|
633 |
+
"time 0.1.43",
|
634 |
"try_from",
|
635 |
"url 1.7.2",
|
636 |
]
|
|
|
826 |
checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331"
|
827 |
dependencies = [
|
828 |
"quote 1.0.33",
|
829 |
+
"syn 2.0.32",
|
830 |
+
]
|
831 |
+
|
832 |
+
[[package]]
|
833 |
+
name = "dashmap"
|
834 |
+
version = "5.5.3"
|
835 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
836 |
+
checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856"
|
837 |
+
dependencies = [
|
838 |
+
"cfg-if 1.0.0",
|
839 |
+
"hashbrown 0.14.0",
|
840 |
+
"lock_api 0.4.10",
|
841 |
+
"once_cell",
|
842 |
+
"parking_lot_core 0.9.8",
|
843 |
]
|
844 |
|
845 |
[[package]]
|
|
|
861 |
"syn 1.0.109",
|
862 |
]
|
863 |
|
864 |
+
[[package]]
|
865 |
+
name = "dhat"
|
866 |
+
version = "0.3.2"
|
867 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
868 |
+
checksum = "4f2aaf837aaf456f6706cb46386ba8dffd4013a757e36f4ea05c20dd46b209a3"
|
869 |
+
dependencies = [
|
870 |
+
"backtrace",
|
871 |
+
"lazy_static",
|
872 |
+
"mintex",
|
873 |
+
"parking_lot 0.12.1",
|
874 |
+
"rustc-hash",
|
875 |
+
"serde",
|
876 |
+
"serde_json",
|
877 |
+
"thousands",
|
878 |
+
]
|
879 |
+
|
880 |
[[package]]
|
881 |
name = "digest"
|
882 |
version = "0.10.7"
|
|
|
975 |
|
976 |
[[package]]
|
977 |
name = "error-stack"
|
978 |
+
version = "0.4.1"
|
979 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
980 |
+
checksum = "27a72baa257b5e0e2de241967bc5ee8f855d6072351042688621081d66b2a76b"
|
981 |
dependencies = [
|
982 |
"anyhow",
|
983 |
"rustc_version 0.4.0",
|
|
|
1106 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1107 |
checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678"
|
1108 |
|
1109 |
+
[[package]]
|
1110 |
+
name = "futures"
|
1111 |
+
version = "0.3.28"
|
1112 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1113 |
+
checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40"
|
1114 |
+
dependencies = [
|
1115 |
+
"futures-channel",
|
1116 |
+
"futures-core",
|
1117 |
+
"futures-executor",
|
1118 |
+
"futures-io",
|
1119 |
+
"futures-sink",
|
1120 |
+
"futures-task",
|
1121 |
+
"futures-util",
|
1122 |
+
]
|
1123 |
+
|
1124 |
[[package]]
|
1125 |
name = "futures-channel"
|
1126 |
version = "0.3.28"
|
|
|
1128 |
checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2"
|
1129 |
dependencies = [
|
1130 |
"futures-core",
|
1131 |
+
"futures-sink",
|
1132 |
]
|
1133 |
|
1134 |
[[package]]
|
|
|
1143 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1144 |
checksum = "ab90cde24b3319636588d0c35fe03b1333857621051837ed769faefb4c2162e4"
|
1145 |
dependencies = [
|
1146 |
+
"futures 0.1.31",
|
1147 |
"num_cpus",
|
1148 |
]
|
1149 |
|
1150 |
+
[[package]]
|
1151 |
+
name = "futures-executor"
|
1152 |
+
version = "0.3.28"
|
1153 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1154 |
+
checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0"
|
1155 |
+
dependencies = [
|
1156 |
+
"futures-core",
|
1157 |
+
"futures-task",
|
1158 |
+
"futures-util",
|
1159 |
+
]
|
1160 |
+
|
1161 |
+
[[package]]
|
1162 |
+
name = "futures-io"
|
1163 |
+
version = "0.3.28"
|
1164 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1165 |
+
checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964"
|
1166 |
+
|
1167 |
+
[[package]]
|
1168 |
+
name = "futures-macro"
|
1169 |
+
version = "0.3.28"
|
1170 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1171 |
+
checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"
|
1172 |
+
dependencies = [
|
1173 |
+
"proc-macro2 1.0.66",
|
1174 |
+
"quote 1.0.33",
|
1175 |
+
"syn 2.0.32",
|
1176 |
+
]
|
1177 |
+
|
1178 |
[[package]]
|
1179 |
name = "futures-sink"
|
1180 |
version = "0.3.28"
|
|
|
1187 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1188 |
checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65"
|
1189 |
|
1190 |
+
[[package]]
|
1191 |
+
name = "futures-timer"
|
1192 |
+
version = "3.0.2"
|
1193 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1194 |
+
checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c"
|
1195 |
+
|
1196 |
[[package]]
|
1197 |
name = "futures-util"
|
1198 |
version = "0.3.28"
|
1199 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1200 |
checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533"
|
1201 |
dependencies = [
|
1202 |
+
"futures-channel",
|
1203 |
"futures-core",
|
1204 |
+
"futures-io",
|
1205 |
+
"futures-macro",
|
1206 |
+
"futures-sink",
|
1207 |
"futures-task",
|
1208 |
+
"memchr",
|
1209 |
"pin-project-lite",
|
1210 |
"pin-utils",
|
1211 |
+
"slab",
|
1212 |
]
|
1213 |
|
1214 |
[[package]]
|
|
|
1256 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1257 |
checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0"
|
1258 |
|
1259 |
+
[[package]]
|
1260 |
+
name = "governor"
|
1261 |
+
version = "0.5.1"
|
1262 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1263 |
+
checksum = "c390a940a5d157878dd057c78680a33ce3415bcd05b4799509ea44210914b4d5"
|
1264 |
+
dependencies = [
|
1265 |
+
"cfg-if 1.0.0",
|
1266 |
+
"dashmap",
|
1267 |
+
"futures 0.3.28",
|
1268 |
+
"futures-timer",
|
1269 |
+
"no-std-compat",
|
1270 |
+
"nonzero_ext",
|
1271 |
+
"parking_lot 0.12.1",
|
1272 |
+
"quanta",
|
1273 |
+
"rand 0.8.5",
|
1274 |
+
"smallvec 1.11.0",
|
1275 |
+
]
|
1276 |
+
|
1277 |
[[package]]
|
1278 |
name = "h2"
|
1279 |
version = "0.1.26"
|
|
|
1283 |
"byteorder",
|
1284 |
"bytes 0.4.12",
|
1285 |
"fnv",
|
1286 |
+
"futures 0.1.31",
|
1287 |
"http 0.1.21",
|
1288 |
"indexmap",
|
1289 |
"log",
|
|
|
1298 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1299 |
checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833"
|
1300 |
dependencies = [
|
1301 |
+
"bytes 1.5.0",
|
1302 |
"fnv",
|
1303 |
"futures-core",
|
1304 |
"futures-sink",
|
|
|
1338 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1339 |
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
|
1340 |
|
1341 |
+
[[package]]
|
1342 |
+
name = "hashbrown"
|
1343 |
+
version = "0.14.0"
|
1344 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1345 |
+
checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a"
|
1346 |
+
|
1347 |
[[package]]
|
1348 |
name = "hermit-abi"
|
1349 |
version = "0.3.2"
|
|
|
1395 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1396 |
checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482"
|
1397 |
dependencies = [
|
1398 |
+
"bytes 1.5.0",
|
1399 |
"fnv",
|
1400 |
"itoa 1.0.9",
|
1401 |
]
|
|
|
1407 |
checksum = "6741c859c1b2463a423a1dbce98d418e6c3c3fc720fb0d45528657320920292d"
|
1408 |
dependencies = [
|
1409 |
"bytes 0.4.12",
|
1410 |
+
"futures 0.1.31",
|
1411 |
"http 0.1.21",
|
1412 |
"tokio-buf",
|
1413 |
]
|
|
|
1418 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1419 |
checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1"
|
1420 |
dependencies = [
|
1421 |
+
"bytes 1.5.0",
|
1422 |
"http 0.2.9",
|
1423 |
"pin-project-lite",
|
1424 |
]
|
|
|
1454 |
checksum = "5c843caf6296fc1f93444735205af9ed4e109a539005abb2564ae1d6fad34c52"
|
1455 |
dependencies = [
|
1456 |
"bytes 0.4.12",
|
1457 |
+
"futures 0.1.31",
|
1458 |
"futures-cpupool",
|
1459 |
"h2 0.1.26",
|
1460 |
"http 0.1.21",
|
|
|
1465 |
"log",
|
1466 |
"net2",
|
1467 |
"rustc_version 0.2.3",
|
1468 |
+
"time 0.1.43",
|
1469 |
"tokio 0.1.22",
|
1470 |
"tokio-buf",
|
1471 |
"tokio-executor",
|
|
|
1483 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1484 |
checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468"
|
1485 |
dependencies = [
|
1486 |
+
"bytes 1.5.0",
|
1487 |
"futures-channel",
|
1488 |
"futures-core",
|
1489 |
"futures-util",
|
|
|
1508 |
checksum = "3a800d6aa50af4b5850b2b0f659625ce9504df908e9733b635720483be26174f"
|
1509 |
dependencies = [
|
1510 |
"bytes 0.4.12",
|
1511 |
+
"futures 0.1.31",
|
1512 |
"hyper 0.12.36",
|
1513 |
"native-tls",
|
1514 |
"tokio-io",
|
|
|
1520 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1521 |
checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
|
1522 |
dependencies = [
|
1523 |
+
"bytes 1.5.0",
|
1524 |
"hyper 0.14.27",
|
1525 |
"native-tls",
|
1526 |
"tokio 1.32.0",
|
|
|
1566 |
checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
|
1567 |
dependencies = [
|
1568 |
"autocfg 1.1.0",
|
1569 |
+
"hashbrown 0.12.3",
|
1570 |
]
|
1571 |
|
1572 |
[[package]]
|
|
|
1662 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1663 |
checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3"
|
1664 |
|
1665 |
+
[[package]]
|
1666 |
+
name = "libmimalloc-sys"
|
1667 |
+
version = "0.1.34"
|
1668 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1669 |
+
checksum = "25d058a81af0d1c22d7a1c948576bee6d673f7af3c0f35564abd6c81122f513d"
|
1670 |
+
dependencies = [
|
1671 |
+
"cc",
|
1672 |
+
"libc",
|
1673 |
+
]
|
1674 |
+
|
1675 |
[[package]]
|
1676 |
name = "linux-raw-sys"
|
1677 |
+
version = "0.4.7"
|
1678 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1679 |
+
checksum = "1a9bad9f94746442c783ca431b22403b519cd7fbeed0533fdd6328b2f2212128"
|
1680 |
|
1681 |
[[package]]
|
1682 |
name = "local-channel"
|
|
|
1727 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1728 |
checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
|
1729 |
|
1730 |
+
[[package]]
|
1731 |
+
name = "mach"
|
1732 |
+
version = "0.3.2"
|
1733 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1734 |
+
checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa"
|
1735 |
+
dependencies = [
|
1736 |
+
"libc",
|
1737 |
+
]
|
1738 |
+
|
1739 |
[[package]]
|
1740 |
name = "markup5ever"
|
1741 |
version = "0.8.1"
|
|
|
1809 |
"autocfg 1.1.0",
|
1810 |
]
|
1811 |
|
1812 |
+
[[package]]
|
1813 |
+
name = "mimalloc"
|
1814 |
+
version = "0.1.38"
|
1815 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1816 |
+
checksum = "972e5f23f6716f62665760b0f4cbf592576a80c7b879ba9beaafc0e558894127"
|
1817 |
+
dependencies = [
|
1818 |
+
"libmimalloc-sys",
|
1819 |
+
]
|
1820 |
+
|
1821 |
[[package]]
|
1822 |
name = "mime"
|
1823 |
version = "0.3.17"
|
|
|
1843 |
"adler",
|
1844 |
]
|
1845 |
|
1846 |
+
[[package]]
|
1847 |
+
name = "mintex"
|
1848 |
+
version = "0.1.2"
|
1849 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1850 |
+
checksum = "fd7c5ba1c3b5a23418d7bbf98c71c3d4946a0125002129231da8d6b723d559cb"
|
1851 |
+
dependencies = [
|
1852 |
+
"once_cell",
|
1853 |
+
"sys-info",
|
1854 |
+
]
|
1855 |
+
|
1856 |
[[package]]
|
1857 |
name = "mio"
|
1858 |
version = "0.6.23"
|
|
|
1896 |
"ws2_32-sys",
|
1897 |
]
|
1898 |
|
1899 |
+
[[package]]
|
1900 |
+
name = "mlua"
|
1901 |
+
version = "0.8.10"
|
1902 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1903 |
+
checksum = "0bb37b0ba91f017aa7ca2b98ef99496827770cd635b4a932a6047c5b4bbe678e"
|
1904 |
+
dependencies = [
|
1905 |
+
"bstr",
|
1906 |
+
"cc",
|
1907 |
+
"num-traits",
|
1908 |
+
"once_cell",
|
1909 |
+
"pkg-config",
|
1910 |
+
"rustc-hash",
|
1911 |
+
]
|
1912 |
+
|
1913 |
[[package]]
|
1914 |
name = "native-tls"
|
1915 |
version = "0.2.11"
|
|
|
1951 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1952 |
checksum = "ab250442c86f1850815b5d268639dff018c0627022bc1940eb2d642ca1ce12f0"
|
1953 |
|
1954 |
+
[[package]]
|
1955 |
+
name = "no-std-compat"
|
1956 |
+
version = "0.4.1"
|
1957 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1958 |
+
checksum = "b93853da6d84c2e3c7d730d6473e8817692dd89be387eb01b94d7f108ecb5b8c"
|
1959 |
+
|
1960 |
+
[[package]]
|
1961 |
+
name = "nonzero_ext"
|
1962 |
+
version = "0.3.0"
|
1963 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1964 |
+
checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21"
|
1965 |
+
|
1966 |
[[package]]
|
1967 |
name = "num-traits"
|
1968 |
version = "0.2.16"
|
|
|
2026 |
dependencies = [
|
2027 |
"proc-macro2 1.0.66",
|
2028 |
"quote 1.0.33",
|
2029 |
+
"syn 2.0.32",
|
2030 |
]
|
2031 |
|
2032 |
[[package]]
|
|
|
2037 |
|
2038 |
[[package]]
|
2039 |
name = "openssl-sys"
|
2040 |
+
version = "0.9.93"
|
2041 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
2042 |
+
checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d"
|
2043 |
dependencies = [
|
2044 |
"cc",
|
2045 |
"libc",
|
|
|
2145 |
"pest_meta",
|
2146 |
"proc-macro2 1.0.66",
|
2147 |
"quote 1.0.33",
|
2148 |
+
"syn 2.0.32",
|
2149 |
]
|
2150 |
|
2151 |
[[package]]
|
|
|
2247 |
"phf_shared 0.11.2",
|
2248 |
"proc-macro2 1.0.66",
|
2249 |
"quote 1.0.33",
|
2250 |
+
"syn 2.0.32",
|
2251 |
]
|
2252 |
|
2253 |
[[package]]
|
|
|
2277 |
"siphasher 0.3.11",
|
2278 |
]
|
2279 |
|
2280 |
+
[[package]]
|
2281 |
+
name = "pin-project"
|
2282 |
+
version = "1.1.3"
|
2283 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
2284 |
+
checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422"
|
2285 |
+
dependencies = [
|
2286 |
+
"pin-project-internal",
|
2287 |
+
]
|
2288 |
+
|
2289 |
+
[[package]]
|
2290 |
+
name = "pin-project-internal"
|
2291 |
+
version = "1.1.3"
|
2292 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
2293 |
+
checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"
|
2294 |
+
dependencies = [
|
2295 |
+
"proc-macro2 1.0.66",
|
2296 |
+
"quote 1.0.33",
|
2297 |
+
"syn 2.0.32",
|
2298 |
+
]
|
2299 |
+
|
2300 |
[[package]]
|
2301 |
name = "pin-project-lite"
|
2302 |
version = "0.2.13"
|
|
|
2383 |
"url 2.4.1",
|
2384 |
]
|
2385 |
|
2386 |
+
[[package]]
|
2387 |
+
name = "quanta"
|
2388 |
+
version = "0.9.3"
|
2389 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
2390 |
+
checksum = "20afe714292d5e879d8b12740aa223c6a88f118af41870e8b6196e39a02238a8"
|
2391 |
+
dependencies = [
|
2392 |
+
"crossbeam-utils 0.8.16",
|
2393 |
+
"libc",
|
2394 |
+
"mach",
|
2395 |
+
"once_cell",
|
2396 |
+
"raw-cpuid",
|
2397 |
+
"wasi 0.10.2+wasi-snapshot-preview1",
|
2398 |
+
"web-sys",
|
2399 |
+
"winapi 0.3.9",
|
2400 |
+
]
|
2401 |
+
|
2402 |
[[package]]
|
2403 |
name = "quote"
|
2404 |
version = "0.6.13"
|
|
|
2553 |
"rand_core 0.3.1",
|
2554 |
]
|
2555 |
|
2556 |
+
[[package]]
|
2557 |
+
name = "raw-cpuid"
|
2558 |
+
version = "10.7.0"
|
2559 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
2560 |
+
checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332"
|
2561 |
+
dependencies = [
|
2562 |
+
"bitflags 1.3.2",
|
2563 |
+
]
|
2564 |
+
|
2565 |
[[package]]
|
2566 |
name = "rayon"
|
2567 |
version = "1.7.0"
|
|
|
2599 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
2600 |
checksum = "4f49cdc0bb3f412bf8e7d1bd90fe1d9eb10bc5c399ba90973c14662a27b3f8ba"
|
2601 |
dependencies = [
|
2602 |
+
"arc-swap",
|
2603 |
+
"async-trait",
|
2604 |
+
"bytes 1.5.0",
|
2605 |
"combine",
|
2606 |
+
"futures 0.3.28",
|
2607 |
+
"futures-util",
|
2608 |
"itoa 1.0.9",
|
2609 |
"percent-encoding 2.3.0",
|
2610 |
+
"pin-project-lite",
|
2611 |
"ryu",
|
2612 |
"sha1_smol",
|
2613 |
"socket2 0.4.9",
|
2614 |
+
"tokio 1.32.0",
|
2615 |
+
"tokio-retry",
|
2616 |
+
"tokio-util",
|
2617 |
"url 2.4.1",
|
2618 |
]
|
2619 |
|
|
|
2673 |
"cookie_store",
|
2674 |
"encoding_rs",
|
2675 |
"flate2",
|
2676 |
+
"futures 0.1.31",
|
2677 |
"http 0.1.21",
|
2678 |
"hyper 0.12.36",
|
2679 |
"hyper-tls 0.3.2",
|
|
|
2684 |
"serde",
|
2685 |
"serde_json",
|
2686 |
"serde_urlencoded 0.5.5",
|
2687 |
+
"time 0.1.43",
|
2688 |
"tokio 0.1.22",
|
2689 |
"tokio-executor",
|
2690 |
"tokio-io",
|
|
|
2701 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
2702 |
checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1"
|
2703 |
dependencies = [
|
2704 |
+
"base64 0.21.4",
|
2705 |
+
"bytes 1.5.0",
|
2706 |
"encoding_rs",
|
2707 |
"futures-core",
|
2708 |
"futures-util",
|
|
|
2732 |
"winreg 0.50.0",
|
2733 |
]
|
2734 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2735 |
[[package]]
|
2736 |
name = "rustc-demangle"
|
2737 |
version = "0.1.23"
|
2738 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
2739 |
checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
|
2740 |
|
2741 |
+
[[package]]
|
2742 |
+
name = "rustc-hash"
|
2743 |
+
version = "1.1.0"
|
2744 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
2745 |
+
checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
|
2746 |
+
|
2747 |
[[package]]
|
2748 |
name = "rustc_version"
|
2749 |
version = "0.2.3"
|
|
|
2764 |
|
2765 |
[[package]]
|
2766 |
name = "rustix"
|
2767 |
+
version = "0.38.13"
|
2768 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
2769 |
+
checksum = "d7db8590df6dfcd144d22afd1b83b36c21a18d7cbc1dc4bb5295a8712e9eb662"
|
2770 |
dependencies = [
|
2771 |
"bitflags 2.4.0",
|
2772 |
"errno",
|
|
|
2924 |
dependencies = [
|
2925 |
"proc-macro2 1.0.66",
|
2926 |
"quote 1.0.33",
|
2927 |
+
"syn 2.0.32",
|
2928 |
]
|
2929 |
|
2930 |
[[package]]
|
2931 |
name = "serde_json"
|
2932 |
+
version = "1.0.106"
|
2933 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
2934 |
+
checksum = "2cc66a619ed80bf7a0f6b17dd063a84b88f6dea1813737cf469aef1d081142c2"
|
2935 |
dependencies = [
|
2936 |
"itoa 1.0.9",
|
2937 |
"ryu",
|
|
|
3043 |
version = "1.11.0"
|
3044 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
3045 |
checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9"
|
3046 |
+
dependencies = [
|
3047 |
+
"serde",
|
3048 |
+
]
|
3049 |
|
3050 |
[[package]]
|
3051 |
name = "socket2"
|
|
|
3166 |
|
3167 |
[[package]]
|
3168 |
name = "syn"
|
3169 |
+
version = "2.0.32"
|
3170 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
3171 |
+
checksum = "239814284fd6f1a4ffe4ca893952cdd93c224b6a1571c9a9eadd670295c0c9e2"
|
3172 |
dependencies = [
|
3173 |
"proc-macro2 1.0.66",
|
3174 |
"quote 1.0.33",
|
|
|
3187 |
"unicode-xid 0.2.4",
|
3188 |
]
|
3189 |
|
3190 |
+
[[package]]
|
3191 |
+
name = "sys-info"
|
3192 |
+
version = "0.9.1"
|
3193 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
3194 |
+
checksum = "0b3a0d0aba8bf96a0e1ddfdc352fc53b3df7f39318c71854910c3c4b024ae52c"
|
3195 |
+
dependencies = [
|
3196 |
+
"cc",
|
3197 |
+
"libc",
|
3198 |
+
]
|
3199 |
+
|
3200 |
[[package]]
|
3201 |
name = "tempfile"
|
3202 |
version = "3.8.0"
|
|
|
3247 |
dependencies = [
|
3248 |
"proc-macro2 1.0.66",
|
3249 |
"quote 1.0.33",
|
3250 |
+
"syn 2.0.32",
|
3251 |
]
|
3252 |
|
3253 |
+
[[package]]
|
3254 |
+
name = "thousands"
|
3255 |
+
version = "0.2.0"
|
3256 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
3257 |
+
checksum = "3bf63baf9f5039dadc247375c29eb13706706cfde997d0330d05aa63a77d8820"
|
3258 |
+
|
3259 |
[[package]]
|
3260 |
name = "time"
|
3261 |
+
version = "0.1.43"
|
3262 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
3263 |
+
checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438"
|
3264 |
dependencies = [
|
3265 |
"libc",
|
|
|
3266 |
"winapi 0.3.9",
|
3267 |
]
|
3268 |
|
|
|
3326 |
checksum = "5a09c0b5bb588872ab2f09afa13ee6e9dac11e10a0ec9e8e3ba39a5a5d530af6"
|
3327 |
dependencies = [
|
3328 |
"bytes 0.4.12",
|
3329 |
+
"futures 0.1.31",
|
3330 |
"mio 0.6.23",
|
3331 |
"num_cpus",
|
3332 |
"tokio-current-thread",
|
|
|
3345 |
checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9"
|
3346 |
dependencies = [
|
3347 |
"backtrace",
|
3348 |
+
"bytes 1.5.0",
|
3349 |
"libc",
|
3350 |
"mio 0.8.8",
|
3351 |
"num_cpus",
|
|
|
3365 |
dependencies = [
|
3366 |
"bytes 0.4.12",
|
3367 |
"either",
|
3368 |
+
"futures 0.1.31",
|
3369 |
]
|
3370 |
|
3371 |
[[package]]
|
|
|
3374 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
3375 |
checksum = "b1de0e32a83f131e002238d7ccde18211c0a5397f60cbfffcb112868c2e0e20e"
|
3376 |
dependencies = [
|
3377 |
+
"futures 0.1.31",
|
3378 |
"tokio-executor",
|
3379 |
]
|
3380 |
|
|
|
3385 |
checksum = "fb2d1b8f4548dbf5e1f7818512e9c406860678f29c300cdf0ebac72d1a3a1671"
|
3386 |
dependencies = [
|
3387 |
"crossbeam-utils 0.7.2",
|
3388 |
+
"futures 0.1.31",
|
3389 |
]
|
3390 |
|
3391 |
[[package]]
|
|
|
3395 |
checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674"
|
3396 |
dependencies = [
|
3397 |
"bytes 0.4.12",
|
3398 |
+
"futures 0.1.31",
|
3399 |
"log",
|
3400 |
]
|
3401 |
|
|
|
3407 |
dependencies = [
|
3408 |
"proc-macro2 1.0.66",
|
3409 |
"quote 1.0.33",
|
3410 |
+
"syn 2.0.32",
|
3411 |
]
|
3412 |
|
3413 |
[[package]]
|
|
|
3427 |
checksum = "09bc590ec4ba8ba87652da2068d150dcada2cfa2e07faae270a5e0409aa51351"
|
3428 |
dependencies = [
|
3429 |
"crossbeam-utils 0.7.2",
|
3430 |
+
"futures 0.1.31",
|
3431 |
"lazy_static",
|
3432 |
"log",
|
3433 |
"mio 0.6.23",
|
|
|
3439 |
"tokio-sync",
|
3440 |
]
|
3441 |
|
3442 |
+
[[package]]
|
3443 |
+
name = "tokio-retry"
|
3444 |
+
version = "0.3.0"
|
3445 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
3446 |
+
checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f"
|
3447 |
+
dependencies = [
|
3448 |
+
"pin-project",
|
3449 |
+
"rand 0.8.5",
|
3450 |
+
"tokio 1.32.0",
|
3451 |
+
]
|
3452 |
+
|
3453 |
[[package]]
|
3454 |
name = "tokio-sync"
|
3455 |
version = "0.1.8"
|
|
|
3457 |
checksum = "edfe50152bc8164fcc456dab7891fa9bf8beaf01c5ee7e1dd43a397c3cf87dee"
|
3458 |
dependencies = [
|
3459 |
"fnv",
|
3460 |
+
"futures 0.1.31",
|
3461 |
]
|
3462 |
|
3463 |
[[package]]
|
|
|
3467 |
checksum = "98df18ed66e3b72e742f185882a9e201892407957e45fbff8da17ae7a7c51f72"
|
3468 |
dependencies = [
|
3469 |
"bytes 0.4.12",
|
3470 |
+
"futures 0.1.31",
|
3471 |
"iovec",
|
3472 |
"mio 0.6.23",
|
3473 |
"tokio-io",
|
|
|
3483 |
"crossbeam-deque 0.7.4",
|
3484 |
"crossbeam-queue",
|
3485 |
"crossbeam-utils 0.7.2",
|
3486 |
+
"futures 0.1.31",
|
3487 |
"lazy_static",
|
3488 |
"log",
|
3489 |
"num_cpus",
|
|
|
3498 |
checksum = "93044f2d313c95ff1cb7809ce9a7a05735b012288a888b62d4434fd58c94f296"
|
3499 |
dependencies = [
|
3500 |
"crossbeam-utils 0.7.2",
|
3501 |
+
"futures 0.1.31",
|
3502 |
"slab",
|
3503 |
"tokio-executor",
|
3504 |
]
|
|
|
3509 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
3510 |
checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d"
|
3511 |
dependencies = [
|
3512 |
+
"bytes 1.5.0",
|
3513 |
"futures-core",
|
3514 |
"futures-sink",
|
3515 |
"pin-project-lite",
|
|
|
3679 |
|
3680 |
[[package]]
|
3681 |
name = "walkdir"
|
3682 |
+
version = "2.4.0"
|
3683 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
3684 |
+
checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee"
|
3685 |
dependencies = [
|
3686 |
"same-file",
|
3687 |
"winapi-util",
|
|
|
3693 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
3694 |
checksum = "b6395efa4784b027708f7451087e647ec73cc74f5d9bc2e418404248d679a230"
|
3695 |
dependencies = [
|
3696 |
+
"futures 0.1.31",
|
3697 |
"log",
|
3698 |
"try-lock",
|
3699 |
]
|
|
|
3709 |
|
3710 |
[[package]]
|
3711 |
name = "wasi"
|
3712 |
+
version = "0.10.2+wasi-snapshot-preview1"
|
3713 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
3714 |
+
checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
|
3715 |
|
3716 |
[[package]]
|
3717 |
name = "wasi"
|
|
|
3740 |
"once_cell",
|
3741 |
"proc-macro2 1.0.66",
|
3742 |
"quote 1.0.33",
|
3743 |
+
"syn 2.0.32",
|
3744 |
"wasm-bindgen-shared",
|
3745 |
]
|
3746 |
|
|
|
3774 |
dependencies = [
|
3775 |
"proc-macro2 1.0.66",
|
3776 |
"quote 1.0.33",
|
3777 |
+
"syn 2.0.32",
|
3778 |
"wasm-bindgen-backend",
|
3779 |
"wasm-bindgen-shared",
|
3780 |
]
|
|
|
3801 |
dependencies = [
|
3802 |
"actix-cors",
|
3803 |
"actix-files",
|
3804 |
+
"actix-governor",
|
3805 |
"actix-web",
|
3806 |
+
"async-once-cell",
|
3807 |
"async-trait",
|
3808 |
"criterion",
|
3809 |
+
"dhat",
|
3810 |
"env_logger",
|
3811 |
"error-stack",
|
3812 |
"fake-useragent",
|
3813 |
+
"futures 0.3.28",
|
3814 |
"handlebars",
|
3815 |
"log",
|
3816 |
"md5",
|
3817 |
+
"mimalloc",
|
3818 |
+
"mlua",
|
3819 |
"once_cell",
|
3820 |
"rand 0.8.5",
|
3821 |
"redis",
|
3822 |
"regex",
|
3823 |
"reqwest 0.11.20",
|
|
|
3824 |
"rusty-hook",
|
3825 |
"scraper",
|
3826 |
"serde",
|
3827 |
"serde_json",
|
3828 |
+
"smallvec 1.11.0",
|
3829 |
"tempfile",
|
3830 |
"tokio 1.32.0",
|
3831 |
]
|
Cargo.toml
CHANGED
@@ -8,9 +8,9 @@ license = "AGPL-3.0"
|
|
8 |
|
9 |
[dependencies]
|
10 |
reqwest = {version="0.11.20",features=["json"]}
|
11 |
-
tokio = {version="1.32.0",features=["
|
12 |
serde = {version="1.0.188",features=["derive"]}
|
13 |
-
handlebars = { version = "4.
|
14 |
scraper = {version="0.17.1"}
|
15 |
actix-web = {version="4.4.0", features = ["cookies"]}
|
16 |
actix-files = {version="0.6.2"}
|
@@ -19,14 +19,20 @@ serde_json = {version="1.0.105"}
|
|
19 |
fake-useragent = {version="0.1.3"}
|
20 |
env_logger = {version="0.10.0"}
|
21 |
log = {version="0.4.20"}
|
22 |
-
|
23 |
-
redis = {version="0.23.
|
24 |
md5 = {version="0.7.0"}
|
25 |
rand={version="0.8.5"}
|
26 |
once_cell = {version="1.18.0"}
|
27 |
error-stack = {version="0.4.0"}
|
28 |
async-trait = {version="0.1.73"}
|
29 |
regex = {version="1.9.4", features=["perf"]}
|
|
|
|
|
|
|
|
|
|
|
|
|
30 |
|
31 |
[dev-dependencies]
|
32 |
rusty-hook = "^0.11.2"
|
@@ -47,13 +53,17 @@ rpath = false
|
|
47 |
|
48 |
[profile.release]
|
49 |
opt-level = 3
|
50 |
-
debug = false
|
|
|
51 |
split-debuginfo = '...'
|
52 |
debug-assertions = false
|
53 |
overflow-checks = false
|
54 |
-
lto =
|
55 |
panic = 'abort'
|
56 |
incremental = false
|
57 |
-
codegen-units =
|
58 |
rpath = false
|
59 |
strip = "debuginfo"
|
|
|
|
|
|
|
|
8 |
|
9 |
[dependencies]
|
10 |
reqwest = {version="0.11.20",features=["json"]}
|
11 |
+
tokio = {version="1.32.0",features=["rt-multi-thread","macros"]}
|
12 |
serde = {version="1.0.188",features=["derive"]}
|
13 |
+
handlebars = { version = "4.4.0", features = ["dir_source"] }
|
14 |
scraper = {version="0.17.1"}
|
15 |
actix-web = {version="4.4.0", features = ["cookies"]}
|
16 |
actix-files = {version="0.6.2"}
|
|
|
19 |
fake-useragent = {version="0.1.3"}
|
20 |
env_logger = {version="0.10.0"}
|
21 |
log = {version="0.4.20"}
|
22 |
+
mlua = {version="0.8.10", features=["luajit"]}
|
23 |
+
redis = {version="0.23.3", features=["tokio-comp","connection-manager"]}
|
24 |
md5 = {version="0.7.0"}
|
25 |
rand={version="0.8.5"}
|
26 |
once_cell = {version="1.18.0"}
|
27 |
error-stack = {version="0.4.0"}
|
28 |
async-trait = {version="0.1.73"}
|
29 |
regex = {version="1.9.4", features=["perf"]}
|
30 |
+
smallvec = {version="1.11.0", features=["union", "serde"]}
|
31 |
+
futures = {version="0.3.28"}
|
32 |
+
dhat = {version="0.3.2", optional = true}
|
33 |
+
mimalloc = { version = "0.1.38", default-features = false }
|
34 |
+
async-once-cell = {version="0.5.3"}
|
35 |
+
actix-governor = {version="0.4.1"}
|
36 |
|
37 |
[dev-dependencies]
|
38 |
rusty-hook = "^0.11.2"
|
|
|
53 |
|
54 |
[profile.release]
|
55 |
opt-level = 3
|
56 |
+
debug = false # This should only be commented when testing with dhat profiler
|
57 |
+
# debug = 1 # This should only be uncommented when testing with dhat profiler
|
58 |
split-debuginfo = '...'
|
59 |
debug-assertions = false
|
60 |
overflow-checks = false
|
61 |
+
lto = true
|
62 |
panic = 'abort'
|
63 |
incremental = false
|
64 |
+
codegen-units = 1
|
65 |
rpath = false
|
66 |
strip = "debuginfo"
|
67 |
+
|
68 |
+
[features]
|
69 |
+
dhat-heap = ["dep:dhat"]
|
Dockerfile
CHANGED
@@ -19,7 +19,7 @@ COPY . .
|
|
19 |
RUN cargo install --path .
|
20 |
|
21 |
# We do not need the Rust toolchain to run the binary!
|
22 |
-
FROM gcr.io/distroless/cc-
|
23 |
COPY --from=builder /app/public/ /opt/websurfx/public/
|
24 |
COPY --from=builder /app/websurfx/config.lua /etc/xdg/websurfx/config.lua
|
25 |
COPY --from=builder /usr/local/cargo/bin/* /usr/local/bin/
|
|
|
19 |
RUN cargo install --path .
|
20 |
|
21 |
# We do not need the Rust toolchain to run the binary!
|
22 |
+
FROM gcr.io/distroless/cc-debian12
|
23 |
COPY --from=builder /app/public/ /opt/websurfx/public/
|
24 |
COPY --from=builder /app/websurfx/config.lua /etc/xdg/websurfx/config.lua
|
25 |
COPY --from=builder /usr/local/cargo/bin/* /usr/local/bin/
|
README.md
CHANGED
@@ -5,7 +5,7 @@
|
|
5 |
<b align="center"><a href="README.md">Readme</a></b> |
|
6 |
<b><a href="https://discord.gg/SWnda7Mw5u">Discord</a></b> |
|
7 |
<b><a href="https://github.com/neon-mmd/websurfx">GitHub</a></b> |
|
8 |
-
<b><a href="
|
9 |
<br /><br />
|
10 |
<a href="#">
|
11 |
<img
|
@@ -51,7 +51,7 @@
|
|
51 |
- **Getting Started**
|
52 |
- [🔭 Preview](#preview-)
|
53 |
- [🚀 Features](#features-)
|
54 |
-
- [🛠️ Installation and Testing](#installation-and-testing
|
55 |
- [🔧 Configuration](#configuration-)
|
56 |
- **Feature Overview**
|
57 |
- [🎨 Theming](#theming-)
|
|
|
5 |
<b align="center"><a href="README.md">Readme</a></b> |
|
6 |
<b><a href="https://discord.gg/SWnda7Mw5u">Discord</a></b> |
|
7 |
<b><a href="https://github.com/neon-mmd/websurfx">GitHub</a></b> |
|
8 |
+
<b><a href="../../tree/HEAD/docs/">Documentation</a></b>
|
9 |
<br /><br />
|
10 |
<a href="#">
|
11 |
<img
|
|
|
51 |
- **Getting Started**
|
52 |
- [🔭 Preview](#preview-)
|
53 |
- [🚀 Features](#features-)
|
54 |
+
- [🛠️ Installation and Testing](#installation-and-testing-%EF%B8%8F)
|
55 |
- [🔧 Configuration](#configuration-)
|
56 |
- **Feature Overview**
|
57 |
- [🎨 Theming](#theming-)
|
docs/installation.md
CHANGED
@@ -109,7 +109,7 @@ colorscheme = "catppuccin-mocha" -- the colorscheme name which should be used fo
|
|
109 |
theme = "simple" -- the theme name which should be used for the website
|
110 |
|
111 |
-- ### Caching ###
|
112 |
-
|
113 |
|
114 |
-- ### Search Engines ###
|
115 |
upstream_search_engines = { DuckDuckGo = true, Searx = false } -- select the upstream search engines from which the results should be fetched.
|
|
|
109 |
theme = "simple" -- the theme name which should be used for the website
|
110 |
|
111 |
-- ### Caching ###
|
112 |
+
redis_url = "redis://redis:6379" -- redis connection url address on which the client should connect on.
|
113 |
|
114 |
-- ### Search Engines ###
|
115 |
upstream_search_engines = { DuckDuckGo = true, Searx = false } -- select the upstream search engines from which the results should be fetched.
|
public/images/barricade.png
ADDED
![]() |
public/images/filter.png
ADDED
![]() |
public/static/themes/simple.css
CHANGED
@@ -132,6 +132,35 @@ body {
|
|
132 |
width: 1.2rem;
|
133 |
height: 1.2rem;
|
134 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
135 |
|
136 |
/* styles for the footer and header */
|
137 |
|
|
|
132 |
width: 1.2rem;
|
133 |
height: 1.2rem;
|
134 |
}
|
135 |
+
.results .result_disallowed,
|
136 |
+
.results .result_filtered {
|
137 |
+
display: flex;
|
138 |
+
justify-content: center;
|
139 |
+
align-items: center;
|
140 |
+
gap: 10rem;
|
141 |
+
font-size: 2rem;
|
142 |
+
color: var(--foreground-color);
|
143 |
+
margin: 0rem 7rem;
|
144 |
+
}
|
145 |
+
|
146 |
+
.results .result_disallowed .user_query,
|
147 |
+
.results .result_filtered .user_query {
|
148 |
+
color: var(--background-color);
|
149 |
+
font-weight: 300;
|
150 |
+
}
|
151 |
+
|
152 |
+
.results .result_disallowed img,
|
153 |
+
.results .result_filtered img {
|
154 |
+
width: 30rem;
|
155 |
+
}
|
156 |
+
|
157 |
+
.results .result_disallowed div,
|
158 |
+
.results .result_filtered div {
|
159 |
+
display: flex;
|
160 |
+
flex-direction: column;
|
161 |
+
gap: 1rem;
|
162 |
+
line-break: strict;
|
163 |
+
}
|
164 |
|
165 |
/* styles for the footer and header */
|
166 |
|
public/templates/search.html
CHANGED
@@ -1,37 +1,69 @@
|
|
1 |
{{>header this.style}}
|
2 |
<main class="results">
|
3 |
-
|
4 |
-
|
5 |
-
|
6 |
-
|
7 |
-
|
8 |
-
|
9 |
-
|
10 |
-
|
11 |
-
|
12 |
-
|
13 |
-
|
14 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
15 |
</div>
|
16 |
-
|
17 |
-
|
18 |
-
|
19 |
-
|
20 |
-
|
21 |
-
<li>Make sure that all words are spelled correctly.</li>
|
22 |
-
<li>Try different keywords.</li>
|
23 |
-
<li>Try more general keywords.</li>
|
24 |
-
</ul>
|
25 |
-
<img src="./images/no_results.gif" alt="Man fishing gif" />
|
26 |
</div>
|
27 |
-
{{/if}}
|
28 |
-
</div>
|
29 |
-
<div class="page_navigation">
|
30 |
-
<button type="button" onclick="navigate_backward()">
|
31 |
-
← previous
|
32 |
-
</button>
|
33 |
-
<button type="button" onclick="navigate_forward()">next →</button>
|
34 |
-
</div>
|
35 |
</main>
|
36 |
<script src="static/index.js"></script>
|
37 |
<script src="static/pagination.js"></script>
|
|
|
1 |
{{>header this.style}}
|
2 |
<main class="results">
|
3 |
+
{{>search_bar this}}
|
4 |
+
<div class="results_aggregated">
|
5 |
+
{{#if results}} {{#each results}}
|
6 |
+
<div class="result">
|
7 |
+
<h1><a href="{{{this.url}}}">{{{this.title}}}</a></h1>
|
8 |
+
<small>{{{this.url}}}</small>
|
9 |
+
<p>{{{this.description}}}</p>
|
10 |
+
<div class="upstream_engines">
|
11 |
+
{{#each engine}}
|
12 |
+
<span>{{{this}}}</span>
|
13 |
+
{{/each}}
|
14 |
+
</div>
|
15 |
+
</div>
|
16 |
+
{{/each}} {{else}} {{#if disallowed}}
|
17 |
+
<div class="result_disallowed">
|
18 |
+
<div class="description">
|
19 |
+
<p>
|
20 |
+
Your search - <span class="user_query">{{{this.pageQuery}}}</span> -
|
21 |
+
has been disallowed.
|
22 |
+
</p>
|
23 |
+
<p class="description_paragraph">Dear user,</p>
|
24 |
+
<p class="description_paragraph">
|
25 |
+
The query - <span class="user_query">{{{this.pageQuery}}}</span> - has
|
26 |
+
been blacklisted via server configuration and hence disallowed by the
|
27 |
+
server. Henceforth no results could be displayed for your query.
|
28 |
+
</p>
|
29 |
+
</div>
|
30 |
+
<img src="./images/barricade.png" alt="Image of a Barricade" />
|
31 |
+
</div>
|
32 |
+
{{else}} {{#if filtered}}
|
33 |
+
<div class="result_filtered">
|
34 |
+
<div class="description">
|
35 |
+
<p>
|
36 |
+
Your search - <span class="user_query">{{{this.pageQuery}}}</span> -
|
37 |
+
has been filtered.
|
38 |
+
</p>
|
39 |
+
<p class="description_paragraph">Dear user,</p>
|
40 |
+
<p class="description_paragraph">
|
41 |
+
All the search results contain results that has been configured to be
|
42 |
+
filtered out via server configuration and henceforth has been
|
43 |
+
completely filtered out.
|
44 |
+
</p>
|
45 |
+
</div>
|
46 |
+
<img src="./images/filter.png" alt="Image of a paper inside a funnel" />
|
47 |
+
</div>
|
48 |
+
{{else}}
|
49 |
+
<div class="result_not_found">
|
50 |
+
<p>Your search - {{{this.pageQuery}}} - did not match any documents.</p>
|
51 |
+
<p class="suggestions">Suggestions:</p>
|
52 |
+
<ul>
|
53 |
+
<li>Make sure that all words are spelled correctly.</li>
|
54 |
+
<li>Try different keywords.</li>
|
55 |
+
<li>Try more general keywords.</li>
|
56 |
+
</ul>
|
57 |
+
<img src="./images/no_results.gif" alt="Man fishing gif" />
|
58 |
+
</div>
|
59 |
+
{{/if}} {{/if}} {{/if}}
|
60 |
</div>
|
61 |
+
<div class="page_navigation">
|
62 |
+
<button type="button" onclick="navigate_backward()">
|
63 |
+
← previous
|
64 |
+
</button>
|
65 |
+
<button type="button" onclick="navigate_forward()">next →</button>
|
|
|
|
|
|
|
|
|
|
|
66 |
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
67 |
</main>
|
68 |
<script src="static/index.js"></script>
|
69 |
<script src="static/pagination.js"></script>
|
src/bin/websurfx.rs
CHANGED
@@ -3,9 +3,19 @@
|
|
3 |
//! This module contains the main function which handles the logging of the application to the
|
4 |
//! stdout and handles the command line arguments provided and launches the `websurfx` server.
|
5 |
|
|
|
6 |
use std::net::TcpListener;
|
7 |
use websurfx::{config::parser::Config, run};
|
8 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
9 |
/// The function that launches the main server and registers all the routes of the website.
|
10 |
///
|
11 |
/// # Error
|
@@ -14,6 +24,10 @@ use websurfx::{config::parser::Config, run};
|
|
14 |
/// available for being used for other applications.
|
15 |
#[actix_web::main]
|
16 |
async fn main() -> std::io::Result<()> {
|
|
|
|
|
|
|
|
|
17 |
// Initialize the parsed config file.
|
18 |
let config = Config::parse(false).unwrap();
|
19 |
|
|
|
3 |
//! This module contains the main function which handles the logging of the application to the
|
4 |
//! stdout and handles the command line arguments provided and launches the `websurfx` server.
|
5 |
|
6 |
+
use mimalloc::MiMalloc;
|
7 |
use std::net::TcpListener;
|
8 |
use websurfx::{config::parser::Config, run};
|
9 |
|
10 |
+
/// A dhat heap memory profiler
|
11 |
+
#[cfg(feature = "dhat-heap")]
|
12 |
+
#[global_allocator]
|
13 |
+
static ALLOC: dhat::Alloc = dhat::Alloc;
|
14 |
+
|
15 |
+
#[cfg(not(feature = "dhat-heap"))]
|
16 |
+
#[global_allocator]
|
17 |
+
static GLOBAL: MiMalloc = MiMalloc;
|
18 |
+
|
19 |
/// The function that launches the main server and registers all the routes of the website.
|
20 |
///
|
21 |
/// # Error
|
|
|
24 |
/// available for being used for other applications.
|
25 |
#[actix_web::main]
|
26 |
async fn main() -> std::io::Result<()> {
|
27 |
+
// A dhat heap profiler initialization.
|
28 |
+
#[cfg(feature = "dhat-heap")]
|
29 |
+
let _profiler = dhat::Profiler::new_heap();
|
30 |
+
|
31 |
// Initialize the parsed config file.
|
32 |
let config = Config::parse(false).unwrap();
|
33 |
|
src/cache/cacher.rs
CHANGED
@@ -1,14 +1,24 @@
|
|
1 |
//! This module provides the functionality to cache the aggregated results fetched and aggregated
|
2 |
//! from the upstream search engines in a json format.
|
3 |
|
|
|
|
|
4 |
use md5::compute;
|
5 |
-
use redis::{
|
|
|
|
|
6 |
|
7 |
/// A named struct which stores the redis Connection url address to which the client will
|
8 |
/// connect to.
|
|
|
9 |
pub struct RedisCache {
|
10 |
-
/// It stores
|
11 |
-
|
|
|
|
|
|
|
|
|
|
|
12 |
}
|
13 |
|
14 |
impl RedisCache {
|
@@ -16,11 +26,25 @@ impl RedisCache {
|
|
16 |
///
|
17 |
/// # Arguments
|
18 |
///
|
19 |
-
/// * `redis_connection_url` - It
|
20 |
-
|
|
|
|
|
|
|
|
|
|
|
21 |
let client = Client::open(redis_connection_url)?;
|
22 |
-
let
|
23 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
24 |
Ok(redis_cache)
|
25 |
}
|
26 |
|
@@ -29,7 +53,7 @@ impl RedisCache {
|
|
29 |
/// # Arguments
|
30 |
///
|
31 |
/// * `url` - It takes an url as string.
|
32 |
-
fn hash_url(url: &str) -> String {
|
33 |
format!("{:?}", compute(url))
|
34 |
}
|
35 |
|
@@ -38,9 +62,42 @@ impl RedisCache {
|
|
38 |
/// # Arguments
|
39 |
///
|
40 |
/// * `url` - It takes an url as a string.
|
41 |
-
pub fn cached_json(&mut self, url: &str) -> Result<String,
|
42 |
-
|
43 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
44 |
}
|
45 |
|
46 |
/// A function which caches the results by using the hashed `url` as the key and
|
@@ -51,21 +108,45 @@ impl RedisCache {
|
|
51 |
///
|
52 |
/// * `json_results` - It takes the json results string as an argument.
|
53 |
/// * `url` - It takes the url as a String.
|
54 |
-
pub fn cache_results(
|
55 |
&mut self,
|
56 |
-
json_results:
|
57 |
url: &str,
|
58 |
-
) -> Result<(),
|
59 |
-
|
60 |
-
|
61 |
-
// put results_json into cache
|
62 |
-
self.connection.set(&hashed_url_string, json_results)?;
|
63 |
|
64 |
-
|
65 |
-
|
66 |
-
.
|
67 |
-
.
|
68 |
|
69 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
70 |
}
|
71 |
}
|
|
|
1 |
//! This module provides the functionality to cache the aggregated results fetched and aggregated
|
2 |
//! from the upstream search engines in a json format.
|
3 |
|
4 |
+
use error_stack::Report;
|
5 |
+
use futures::future::try_join_all;
|
6 |
use md5::compute;
|
7 |
+
use redis::{aio::ConnectionManager, AsyncCommands, Client, RedisError};
|
8 |
+
|
9 |
+
use super::error::PoolError;
|
10 |
|
11 |
/// A named struct which stores the redis Connection url address to which the client will
|
12 |
/// connect to.
|
13 |
+
#[derive(Clone)]
|
14 |
pub struct RedisCache {
|
15 |
+
/// It stores a pool of connections ready to be used.
|
16 |
+
connection_pool: Vec<ConnectionManager>,
|
17 |
+
/// It stores the size of the connection pool (in other words the number of
|
18 |
+
/// connections that should be stored in the pool).
|
19 |
+
pool_size: u8,
|
20 |
+
/// It stores the index of which connection is being used at the moment.
|
21 |
+
current_connection: u8,
|
22 |
}
|
23 |
|
24 |
impl RedisCache {
|
|
|
26 |
///
|
27 |
/// # Arguments
|
28 |
///
|
29 |
+
/// * `redis_connection_url` - It takes the redis Connection url address.
|
30 |
+
/// * `pool_size` - It takes the size of the connection pool (in other words the number of
|
31 |
+
/// connections that should be stored in the pool).
|
32 |
+
pub async fn new(
|
33 |
+
redis_connection_url: &str,
|
34 |
+
pool_size: u8,
|
35 |
+
) -> Result<Self, Box<dyn std::error::Error>> {
|
36 |
let client = Client::open(redis_connection_url)?;
|
37 |
+
let mut tasks: Vec<_> = Vec::new();
|
38 |
+
|
39 |
+
for _ in 0..pool_size {
|
40 |
+
tasks.push(client.get_tokio_connection_manager());
|
41 |
+
}
|
42 |
+
|
43 |
+
let redis_cache = RedisCache {
|
44 |
+
connection_pool: try_join_all(tasks).await?,
|
45 |
+
pool_size,
|
46 |
+
current_connection: Default::default(),
|
47 |
+
};
|
48 |
Ok(redis_cache)
|
49 |
}
|
50 |
|
|
|
53 |
/// # Arguments
|
54 |
///
|
55 |
/// * `url` - It takes an url as string.
|
56 |
+
fn hash_url(&self, url: &str) -> String {
|
57 |
format!("{:?}", compute(url))
|
58 |
}
|
59 |
|
|
|
62 |
/// # Arguments
|
63 |
///
|
64 |
/// * `url` - It takes an url as a string.
|
65 |
+
pub async fn cached_json(&mut self, url: &str) -> Result<String, Report<PoolError>> {
|
66 |
+
self.current_connection = Default::default();
|
67 |
+
let hashed_url_string: &str = &self.hash_url(url);
|
68 |
+
|
69 |
+
let mut result: Result<String, RedisError> = self.connection_pool
|
70 |
+
[self.current_connection as usize]
|
71 |
+
.get(hashed_url_string)
|
72 |
+
.await;
|
73 |
+
|
74 |
+
// Code to check whether the current connection being used is dropped with connection error
|
75 |
+
// or not. if it drops with the connection error then the current connection is replaced
|
76 |
+
// with a new connection from the pool which is then used to run the redis command then
|
77 |
+
// that connection is also checked whether it is dropped or not if it is not then the
|
78 |
+
// result is passed as a `Result` or else the same process repeats again and if all of the
|
79 |
+
// connections in the pool result in connection drop error then a custom pool error is
|
80 |
+
// returned.
|
81 |
+
loop {
|
82 |
+
match result {
|
83 |
+
Err(error) => match error.is_connection_dropped() {
|
84 |
+
true => {
|
85 |
+
self.current_connection += 1;
|
86 |
+
if self.current_connection == self.pool_size {
|
87 |
+
return Err(Report::new(
|
88 |
+
PoolError::PoolExhaustionWithConnectionDropError,
|
89 |
+
));
|
90 |
+
}
|
91 |
+
result = self.connection_pool[self.current_connection as usize]
|
92 |
+
.get(hashed_url_string)
|
93 |
+
.await;
|
94 |
+
continue;
|
95 |
+
}
|
96 |
+
false => return Err(Report::new(PoolError::RedisError(error))),
|
97 |
+
},
|
98 |
+
Ok(res) => return Ok(res),
|
99 |
+
}
|
100 |
+
}
|
101 |
}
|
102 |
|
103 |
/// A function which caches the results by using the hashed `url` as the key and
|
|
|
108 |
///
|
109 |
/// * `json_results` - It takes the json results string as an argument.
|
110 |
/// * `url` - It takes the url as a String.
|
111 |
+
pub async fn cache_results(
|
112 |
&mut self,
|
113 |
+
json_results: &str,
|
114 |
url: &str,
|
115 |
+
) -> Result<(), Report<PoolError>> {
|
116 |
+
self.current_connection = Default::default();
|
117 |
+
let hashed_url_string: &str = &self.hash_url(url);
|
|
|
|
|
118 |
|
119 |
+
let mut result: Result<(), RedisError> = self.connection_pool
|
120 |
+
[self.current_connection as usize]
|
121 |
+
.set_ex(hashed_url_string, json_results, 60)
|
122 |
+
.await;
|
123 |
|
124 |
+
// Code to check whether the current connection being used is dropped with connection error
|
125 |
+
// or not. if it drops with the connection error then the current connection is replaced
|
126 |
+
// with a new connection from the pool which is then used to run the redis command then
|
127 |
+
// that connection is also checked whether it is dropped or not if it is not then the
|
128 |
+
// result is passed as a `Result` or else the same process repeats again and if all of the
|
129 |
+
// connections in the pool result in connection drop error then a custom pool error is
|
130 |
+
// returned.
|
131 |
+
loop {
|
132 |
+
match result {
|
133 |
+
Err(error) => match error.is_connection_dropped() {
|
134 |
+
true => {
|
135 |
+
self.current_connection += 1;
|
136 |
+
if self.current_connection == self.pool_size {
|
137 |
+
return Err(Report::new(
|
138 |
+
PoolError::PoolExhaustionWithConnectionDropError,
|
139 |
+
));
|
140 |
+
}
|
141 |
+
result = self.connection_pool[self.current_connection as usize]
|
142 |
+
.set_ex(hashed_url_string, json_results, 60)
|
143 |
+
.await;
|
144 |
+
continue;
|
145 |
+
}
|
146 |
+
false => return Err(Report::new(PoolError::RedisError(error))),
|
147 |
+
},
|
148 |
+
Ok(_) => return Ok(()),
|
149 |
+
}
|
150 |
+
}
|
151 |
}
|
152 |
}
|
src/cache/error.rs
ADDED
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
//! This module provides the error enum to handle different errors associated while requesting data from
|
2 |
+
//! the redis server using an async connection pool.
|
3 |
+
use std::fmt;
|
4 |
+
|
5 |
+
use redis::RedisError;
|
6 |
+
|
7 |
+
/// A custom error type used for handling redis async pool associated errors.
|
8 |
+
///
|
9 |
+
/// This enum provides variants three different categories of errors:
|
10 |
+
/// * `RedisError` - This variant handles all errors related to `RedisError`,
|
11 |
+
/// * `PoolExhaustionWithConnectionDropError` - This variant handles the error
|
12 |
+
/// which occurs when all the connections in the connection pool return a connection
|
13 |
+
/// dropped redis error.
|
14 |
+
#[derive(Debug)]
|
15 |
+
pub enum PoolError {
|
16 |
+
RedisError(RedisError),
|
17 |
+
PoolExhaustionWithConnectionDropError,
|
18 |
+
}
|
19 |
+
|
20 |
+
impl fmt::Display for PoolError {
|
21 |
+
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
22 |
+
match self {
|
23 |
+
PoolError::RedisError(redis_error) => {
|
24 |
+
if let Some(detail) = redis_error.detail() {
|
25 |
+
write!(f, "{}", detail)
|
26 |
+
} else {
|
27 |
+
write!(f, "")
|
28 |
+
}
|
29 |
+
}
|
30 |
+
PoolError::PoolExhaustionWithConnectionDropError => {
|
31 |
+
write!(
|
32 |
+
f,
|
33 |
+
"Error all connections from the pool dropped with connection error"
|
34 |
+
)
|
35 |
+
}
|
36 |
+
}
|
37 |
+
}
|
38 |
+
}
|
39 |
+
|
40 |
+
impl error_stack::Context for PoolError {}
|
src/cache/mod.rs
CHANGED
@@ -2,3 +2,4 @@
|
|
2 |
//! results fetched and aggregated from the upstream search engines in a json format.
|
3 |
|
4 |
pub mod cacher;
|
|
|
|
2 |
//! results fetched and aggregated from the upstream search engines in a json format.
|
3 |
|
4 |
pub mod cacher;
|
5 |
+
pub mod error;
|
src/config/parser.rs
CHANGED
@@ -3,9 +3,9 @@
|
|
3 |
|
4 |
use crate::handler::paths::{file_path, FileType};
|
5 |
|
6 |
-
use super::parser_models::Style;
|
7 |
use log::LevelFilter;
|
8 |
-
use
|
9 |
use std::{collections::HashMap, fs, thread::available_parallelism};
|
10 |
|
11 |
/// A named struct which stores the parsed config file options.
|
@@ -32,14 +32,11 @@ pub struct Config {
|
|
32 |
pub request_timeout: u8,
|
33 |
/// It stores the number of threads which controls the app will use to run.
|
34 |
pub threads: u8,
|
35 |
-
|
36 |
-
|
37 |
-
///
|
38 |
-
|
39 |
-
pub
|
40 |
-
/// It stores the option to whether enable or disable random delays between
|
41 |
-
/// requests.
|
42 |
-
pub random_delay: bool,
|
43 |
}
|
44 |
|
45 |
impl Config {
|
@@ -57,53 +54,70 @@ impl Config {
|
|
57 |
/// or io error if the config.lua file doesn't exists otherwise it returns a newly constructed
|
58 |
/// Config struct with all the parsed config options from the parsed config file.
|
59 |
pub fn parse(logging_initialized: bool) -> Result<Self, Box<dyn std::error::Error>> {
|
60 |
-
Lua::new()
|
61 |
-
|
62 |
|
63 |
-
|
64 |
-
|
65 |
-
.exec()?;
|
66 |
|
67 |
-
|
68 |
|
69 |
-
|
70 |
-
|
71 |
|
72 |
-
|
73 |
-
|
74 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
75 |
|
76 |
-
|
77 |
-
|
78 |
-
|
79 |
-
|
80 |
-
|
81 |
-
|
82 |
-
|
83 |
-
|
|
|
|
|
|
|
84 |
|
85 |
-
|
86 |
-
|
87 |
-
|
88 |
-
|
89 |
-
|
90 |
-
|
91 |
-
|
92 |
-
|
93 |
-
|
94 |
-
|
95 |
-
|
96 |
-
|
97 |
-
|
98 |
-
|
99 |
-
|
100 |
-
|
101 |
-
|
102 |
-
|
103 |
-
|
104 |
-
|
105 |
-
|
106 |
-
|
|
|
|
|
|
|
|
|
107 |
})
|
108 |
}
|
109 |
}
|
|
|
3 |
|
4 |
use crate::handler::paths::{file_path, FileType};
|
5 |
|
6 |
+
use super::parser_models::{AggregatorConfig, RateLimiter, Style};
|
7 |
use log::LevelFilter;
|
8 |
+
use mlua::Lua;
|
9 |
use std::{collections::HashMap, fs, thread::available_parallelism};
|
10 |
|
11 |
/// A named struct which stores the parsed config file options.
|
|
|
32 |
pub request_timeout: u8,
|
33 |
/// It stores the number of threads which controls the app will use to run.
|
34 |
pub threads: u8,
|
35 |
+
/// It stores configuration options for the ratelimiting middleware.
|
36 |
+
pub rate_limiter: RateLimiter,
|
37 |
+
/// It stores the level of safe search to be used for restricting content in the
|
38 |
+
/// search results.
|
39 |
+
pub safe_search: u8,
|
|
|
|
|
|
|
40 |
}
|
41 |
|
42 |
impl Config {
|
|
|
54 |
/// or io error if the config.lua file doesn't exists otherwise it returns a newly constructed
|
55 |
/// Config struct with all the parsed config options from the parsed config file.
|
56 |
pub fn parse(logging_initialized: bool) -> Result<Self, Box<dyn std::error::Error>> {
|
57 |
+
let lua = Lua::new();
|
58 |
+
let globals = lua.globals();
|
59 |
|
60 |
+
lua.load(&fs::read_to_string(file_path(FileType::Config)?)?)
|
61 |
+
.exec()?;
|
|
|
62 |
|
63 |
+
let parsed_threads: u8 = globals.get::<_, u8>("threads")?;
|
64 |
|
65 |
+
let debug: bool = globals.get::<_, bool>("debug")?;
|
66 |
+
let logging: bool = globals.get::<_, bool>("logging")?;
|
67 |
|
68 |
+
if !logging_initialized {
|
69 |
+
set_logging_level(debug, logging);
|
70 |
+
}
|
71 |
+
|
72 |
+
let threads: u8 = if parsed_threads == 0 {
|
73 |
+
let total_num_of_threads: usize = available_parallelism()?.get() / 2;
|
74 |
+
log::error!(
|
75 |
+
"Config Error: The value of `threads` option should be a non zero positive integer"
|
76 |
+
);
|
77 |
+
log::error!("Falling back to using {} threads", total_num_of_threads);
|
78 |
+
total_num_of_threads as u8
|
79 |
+
} else {
|
80 |
+
parsed_threads
|
81 |
+
};
|
82 |
|
83 |
+
let rate_limiter = globals.get::<_, HashMap<String, u8>>("rate_limiter")?;
|
84 |
+
|
85 |
+
let parsed_safe_search: u8 = globals.get::<_, u8>("safe_search")?;
|
86 |
+
let safe_search: u8 = match parsed_safe_search {
|
87 |
+
0..=4 => parsed_safe_search,
|
88 |
+
_ => {
|
89 |
+
log::error!("Config Error: The value of `safe_search` option should be a non zero positive integer from 0 to 4.");
|
90 |
+
log::error!("Falling back to using the value `1` for the option");
|
91 |
+
1
|
92 |
+
}
|
93 |
+
};
|
94 |
|
95 |
+
Ok(Config {
|
96 |
+
port: globals.get::<_, u16>("port")?,
|
97 |
+
binding_ip: globals.get::<_, String>("binding_ip")?,
|
98 |
+
style: Style::new(
|
99 |
+
globals.get::<_, String>("theme")?,
|
100 |
+
globals.get::<_, String>("colorscheme")?,
|
101 |
+
),
|
102 |
+
redis_url: globals.get::<_, String>("redis_url")?,
|
103 |
+
aggregator: AggregatorConfig {
|
104 |
+
random_delay: globals.get::<_, bool>("production_use")?,
|
105 |
+
},
|
106 |
+
logging,
|
107 |
+
debug,
|
108 |
+
upstream_search_engines: globals
|
109 |
+
.get::<_, HashMap<String, bool>>("upstream_search_engines")?
|
110 |
+
.into_iter()
|
111 |
+
.filter_map(|(key, value)| value.then_some(key))
|
112 |
+
.filter_map(|engine| crate::engines::engine_models::EngineHandler::new(&engine))
|
113 |
+
.collect(),
|
114 |
+
request_timeout: globals.get::<_, u8>("request_timeout")?,
|
115 |
+
threads,
|
116 |
+
rate_limiter: RateLimiter {
|
117 |
+
number_of_requests: rate_limiter["number_of_requests"],
|
118 |
+
time_limit: rate_limiter["time_limit"],
|
119 |
+
},
|
120 |
+
safe_search,
|
121 |
})
|
122 |
}
|
123 |
}
|
src/config/parser_models.rs
CHANGED
@@ -12,7 +12,7 @@ use serde::{Deserialize, Serialize};
|
|
12 |
/// order to allow the deserializing the json back to struct in aggregate function in
|
13 |
/// aggregator.rs and create a new struct out of it and then serialize it back to json and pass
|
14 |
/// it to the template files.
|
15 |
-
#[derive(Serialize, Deserialize, Clone)]
|
16 |
pub struct Style {
|
17 |
/// It stores the parsed theme option used to set a theme for the website.
|
18 |
pub theme: String,
|
@@ -33,3 +33,20 @@ impl Style {
|
|
33 |
Style { theme, colorscheme }
|
34 |
}
|
35 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
12 |
/// order to allow the deserializing the json back to struct in aggregate function in
|
13 |
/// aggregator.rs and create a new struct out of it and then serialize it back to json and pass
|
14 |
/// it to the template files.
|
15 |
+
#[derive(Serialize, Deserialize, Clone, Default)]
|
16 |
pub struct Style {
|
17 |
/// It stores the parsed theme option used to set a theme for the website.
|
18 |
pub theme: String,
|
|
|
33 |
Style { theme, colorscheme }
|
34 |
}
|
35 |
}
|
36 |
+
|
37 |
+
/// Configuration options for the aggregator.
|
38 |
+
#[derive(Clone)]
|
39 |
+
pub struct AggregatorConfig {
|
40 |
+
/// It stores the option to whether enable or disable random delays between
|
41 |
+
/// requests.
|
42 |
+
pub random_delay: bool,
|
43 |
+
}
|
44 |
+
|
45 |
+
/// Configuration options for the rate limiter middleware.
|
46 |
+
#[derive(Clone)]
|
47 |
+
pub struct RateLimiter {
|
48 |
+
/// The number of request that are allowed within a provided time limit.
|
49 |
+
pub number_of_requests: u8,
|
50 |
+
/// The time limit in which the quantity of requests that should be accepted.
|
51 |
+
pub time_limit: u8,
|
52 |
+
}
|
src/engines/duckduckgo.rs
CHANGED
@@ -4,14 +4,14 @@
|
|
4 |
|
5 |
use std::collections::HashMap;
|
6 |
|
7 |
-
use reqwest::header::
|
8 |
use scraper::{Html, Selector};
|
9 |
|
10 |
use crate::results::aggregation_models::SearchResult;
|
11 |
|
12 |
use super::engine_models::{EngineError, SearchEngine};
|
13 |
|
14 |
-
use error_stack::{
|
15 |
|
16 |
/// A new DuckDuckGo engine type defined in-order to implement the `SearchEngine` trait which allows to
|
17 |
/// reduce code duplication as well as allows to create vector of different search engines easily.
|
@@ -21,10 +21,11 @@ pub struct DuckDuckGo;
|
|
21 |
impl SearchEngine for DuckDuckGo {
|
22 |
async fn results(
|
23 |
&self,
|
24 |
-
query:
|
25 |
page: u32,
|
26 |
-
user_agent:
|
27 |
request_timeout: u8,
|
|
|
28 |
) -> Result<HashMap<String, SearchResult>, EngineError> {
|
29 |
// Page number can be missing or empty string and so appropriate handling is required
|
30 |
// so that upstream server recieves valid page number.
|
@@ -43,38 +44,19 @@ impl SearchEngine for DuckDuckGo {
|
|
43 |
};
|
44 |
|
45 |
// initializing HeaderMap and adding appropriate headers.
|
46 |
-
let
|
47 |
-
|
48 |
-
|
49 |
-
|
50 |
-
.
|
51 |
-
.
|
52 |
-
|
53 |
-
|
54 |
-
|
55 |
-
|
56 |
-
"https://google.com/"
|
57 |
-
.parse()
|
58 |
-
.into_report()
|
59 |
-
.change_context(EngineError::UnexpectedError)?,
|
60 |
-
);
|
61 |
-
header_map.insert(
|
62 |
-
CONTENT_TYPE,
|
63 |
-
"application/x-www-form-urlencoded"
|
64 |
-
.parse()
|
65 |
-
.into_report()
|
66 |
-
.change_context(EngineError::UnexpectedError)?,
|
67 |
-
);
|
68 |
-
header_map.insert(
|
69 |
-
COOKIE,
|
70 |
-
"kl=wt-wt"
|
71 |
-
.parse()
|
72 |
-
.into_report()
|
73 |
-
.change_context(EngineError::UnexpectedError)?,
|
74 |
-
);
|
75 |
|
76 |
let document: Html = Html::parse_document(
|
77 |
-
&DuckDuckGo::fetch_html_from_upstream(self, url, header_map, request_timeout).await?,
|
78 |
);
|
79 |
|
80 |
let no_result: Selector = Selector::parse(".no-results")
|
@@ -108,8 +90,7 @@ impl SearchEngine for DuckDuckGo {
|
|
108 |
.next()
|
109 |
.unwrap()
|
110 |
.inner_html()
|
111 |
-
.trim()
|
112 |
-
.to_string(),
|
113 |
format!(
|
114 |
"https://{}",
|
115 |
result
|
@@ -118,15 +99,15 @@ impl SearchEngine for DuckDuckGo {
|
|
118 |
.unwrap()
|
119 |
.inner_html()
|
120 |
.trim()
|
121 |
-
)
|
|
|
122 |
result
|
123 |
.select(&result_desc)
|
124 |
.next()
|
125 |
.unwrap()
|
126 |
.inner_html()
|
127 |
-
.trim()
|
128 |
-
|
129 |
-
vec!["duckduckgo".to_string()],
|
130 |
)
|
131 |
})
|
132 |
.map(|search_result| (search_result.url.clone(), search_result))
|
|
|
4 |
|
5 |
use std::collections::HashMap;
|
6 |
|
7 |
+
use reqwest::header::HeaderMap;
|
8 |
use scraper::{Html, Selector};
|
9 |
|
10 |
use crate::results::aggregation_models::SearchResult;
|
11 |
|
12 |
use super::engine_models::{EngineError, SearchEngine};
|
13 |
|
14 |
+
use error_stack::{Report, Result, ResultExt};
|
15 |
|
16 |
/// A new DuckDuckGo engine type defined in-order to implement the `SearchEngine` trait which allows to
|
17 |
/// reduce code duplication as well as allows to create vector of different search engines easily.
|
|
|
21 |
impl SearchEngine for DuckDuckGo {
|
22 |
async fn results(
|
23 |
&self,
|
24 |
+
query: &str,
|
25 |
page: u32,
|
26 |
+
user_agent: &str,
|
27 |
request_timeout: u8,
|
28 |
+
_safe_search: u8,
|
29 |
) -> Result<HashMap<String, SearchResult>, EngineError> {
|
30 |
// Page number can be missing or empty string and so appropriate handling is required
|
31 |
// so that upstream server recieves valid page number.
|
|
|
44 |
};
|
45 |
|
46 |
// initializing HeaderMap and adding appropriate headers.
|
47 |
+
let header_map = HeaderMap::try_from(&HashMap::from([
|
48 |
+
("USER_AGENT".to_string(), user_agent.to_string()),
|
49 |
+
("REFERER".to_string(), "https://google.com/".to_string()),
|
50 |
+
(
|
51 |
+
"CONTENT_TYPE".to_string(),
|
52 |
+
"application/x-www-form-urlencoded".to_string(),
|
53 |
+
),
|
54 |
+
("COOKIE".to_string(), "kl=wt-wt".to_string()),
|
55 |
+
]))
|
56 |
+
.change_context(EngineError::UnexpectedError)?;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
57 |
|
58 |
let document: Html = Html::parse_document(
|
59 |
+
&DuckDuckGo::fetch_html_from_upstream(self, &url, header_map, request_timeout).await?,
|
60 |
);
|
61 |
|
62 |
let no_result: Selector = Selector::parse(".no-results")
|
|
|
90 |
.next()
|
91 |
.unwrap()
|
92 |
.inner_html()
|
93 |
+
.trim(),
|
|
|
94 |
format!(
|
95 |
"https://{}",
|
96 |
result
|
|
|
99 |
.unwrap()
|
100 |
.inner_html()
|
101 |
.trim()
|
102 |
+
)
|
103 |
+
.as_str(),
|
104 |
result
|
105 |
.select(&result_desc)
|
106 |
.next()
|
107 |
.unwrap()
|
108 |
.inner_html()
|
109 |
+
.trim(),
|
110 |
+
&["duckduckgo"],
|
|
|
111 |
)
|
112 |
})
|
113 |
.map(|search_result| (search_result.url.clone(), search_result))
|
src/engines/engine_models.rs
CHANGED
@@ -2,7 +2,7 @@
|
|
2 |
//! the upstream search engines with the search query provided by the user.
|
3 |
|
4 |
use crate::results::aggregation_models::SearchResult;
|
5 |
-
use error_stack::{
|
6 |
use std::{collections::HashMap, fmt, time::Duration};
|
7 |
|
8 |
/// A custom error type used for handle engine associated errors.
|
@@ -64,7 +64,7 @@ pub trait SearchEngine: Sync + Send {
|
|
64 |
/// otherwise it returns a custom `EngineError`.
|
65 |
async fn fetch_html_from_upstream(
|
66 |
&self,
|
67 |
-
url:
|
68 |
header_map: reqwest::header::HeaderMap,
|
69 |
request_timeout: u8,
|
70 |
) -> Result<String, EngineError> {
|
@@ -75,11 +75,9 @@ pub trait SearchEngine: Sync + Send {
|
|
75 |
.headers(header_map) // add spoofed headers to emulate human behavior
|
76 |
.send()
|
77 |
.await
|
78 |
-
.into_report()
|
79 |
.change_context(EngineError::RequestError)?
|
80 |
.text()
|
81 |
.await
|
82 |
-
.into_report()
|
83 |
.change_context(EngineError::RequestError)?)
|
84 |
}
|
85 |
|
@@ -103,10 +101,11 @@ pub trait SearchEngine: Sync + Send {
|
|
103 |
/// or HeaderMap fails to initialize.
|
104 |
async fn results(
|
105 |
&self,
|
106 |
-
query:
|
107 |
page: u32,
|
108 |
-
user_agent:
|
109 |
request_timeout: u8,
|
|
|
110 |
) -> Result<HashMap<String, SearchResult>, EngineError>;
|
111 |
}
|
112 |
|
|
|
2 |
//! the upstream search engines with the search query provided by the user.
|
3 |
|
4 |
use crate::results::aggregation_models::SearchResult;
|
5 |
+
use error_stack::{Result, ResultExt};
|
6 |
use std::{collections::HashMap, fmt, time::Duration};
|
7 |
|
8 |
/// A custom error type used for handle engine associated errors.
|
|
|
64 |
/// otherwise it returns a custom `EngineError`.
|
65 |
async fn fetch_html_from_upstream(
|
66 |
&self,
|
67 |
+
url: &str,
|
68 |
header_map: reqwest::header::HeaderMap,
|
69 |
request_timeout: u8,
|
70 |
) -> Result<String, EngineError> {
|
|
|
75 |
.headers(header_map) // add spoofed headers to emulate human behavior
|
76 |
.send()
|
77 |
.await
|
|
|
78 |
.change_context(EngineError::RequestError)?
|
79 |
.text()
|
80 |
.await
|
|
|
81 |
.change_context(EngineError::RequestError)?)
|
82 |
}
|
83 |
|
|
|
101 |
/// or HeaderMap fails to initialize.
|
102 |
async fn results(
|
103 |
&self,
|
104 |
+
query: &str,
|
105 |
page: u32,
|
106 |
+
user_agent: &str,
|
107 |
request_timeout: u8,
|
108 |
+
safe_search: u8,
|
109 |
) -> Result<HashMap<String, SearchResult>, EngineError>;
|
110 |
}
|
111 |
|
src/engines/searx.rs
CHANGED
@@ -2,14 +2,14 @@
|
|
2 |
//! by querying the upstream searx search engine instance with user provided query and with a page
|
3 |
//! number if provided.
|
4 |
|
5 |
-
use reqwest::header::
|
6 |
use scraper::{Html, Selector};
|
7 |
use std::collections::HashMap;
|
8 |
|
9 |
use crate::results::aggregation_models::SearchResult;
|
10 |
|
11 |
use super::engine_models::{EngineError, SearchEngine};
|
12 |
-
use error_stack::{
|
13 |
|
14 |
/// A new Searx engine type defined in-order to implement the `SearchEngine` trait which allows to
|
15 |
/// reduce code duplication as well as allows to create vector of different search engines easily.
|
@@ -19,45 +19,38 @@ pub struct Searx;
|
|
19 |
impl SearchEngine for Searx {
|
20 |
async fn results(
|
21 |
&self,
|
22 |
-
query:
|
23 |
page: u32,
|
24 |
-
user_agent:
|
25 |
request_timeout: u8,
|
|
|
26 |
) -> Result<HashMap<String, SearchResult>, EngineError> {
|
27 |
// Page number can be missing or empty string and so appropriate handling is required
|
28 |
// so that upstream server recieves valid page number.
|
|
|
|
|
|
|
|
|
29 |
let url: String = match page {
|
30 |
-
0 | 1 =>
|
31 |
-
|
|
|
|
|
|
|
|
|
32 |
};
|
33 |
|
34 |
// initializing headers and adding appropriate headers.
|
35 |
-
let
|
36 |
-
|
37 |
-
|
38 |
-
|
39 |
-
|
40 |
-
|
41 |
-
|
42 |
-
);
|
43 |
-
header_map.insert(
|
44 |
-
REFERER,
|
45 |
-
"https://google.com/"
|
46 |
-
.parse()
|
47 |
-
.into_report()
|
48 |
-
.change_context(EngineError::UnexpectedError)?,
|
49 |
-
);
|
50 |
-
header_map.insert(
|
51 |
-
CONTENT_TYPE,
|
52 |
-
"application/x-www-form-urlencoded"
|
53 |
-
.parse()
|
54 |
-
.into_report()
|
55 |
-
.change_context(EngineError::UnexpectedError)?,
|
56 |
-
);
|
57 |
-
header_map.insert(COOKIE, "categories=general; language=auto; locale=en; autocomplete=duckduckgo; image_proxy=1; method=POST; safesearch=2; theme=simple; results_on_new_tab=1; doi_resolver=oadoi.org; simple_style=auto; center_alignment=1; query_in_title=1; infinite_scroll=0; disabled_engines=; enabled_engines=\"archive is__general\\054yep__general\\054curlie__general\\054currency__general\\054ddg definitions__general\\054wikidata__general\\054duckduckgo__general\\054tineye__general\\054lingva__general\\054startpage__general\\054yahoo__general\\054wiby__general\\054marginalia__general\\054alexandria__general\\054wikibooks__general\\054wikiquote__general\\054wikisource__general\\054wikiversity__general\\054wikivoyage__general\\054dictzone__general\\054seznam__general\\054mojeek__general\\054naver__general\\054wikimini__general\\054brave__general\\054petalsearch__general\\054goo__general\"; disabled_plugins=; enabled_plugins=\"searx.plugins.hostname_replace\\054searx.plugins.oa_doi_rewrite\\054searx.plugins.vim_hotkeys\"; tokens=; maintab=on; enginetab=on".parse().into_report().change_context(EngineError::UnexpectedError)?);
|
58 |
|
59 |
let document: Html = Html::parse_document(
|
60 |
-
&Searx::fetch_html_from_upstream(self, url, header_map, request_timeout).await?,
|
61 |
);
|
62 |
|
63 |
let no_result: Selector = Selector::parse("#urls>.dialog-error>p")
|
@@ -98,24 +91,21 @@ impl SearchEngine for Searx {
|
|
98 |
.next()
|
99 |
.unwrap()
|
100 |
.inner_html()
|
101 |
-
.trim()
|
102 |
-
.to_string(),
|
103 |
result
|
104 |
.select(&result_url)
|
105 |
.next()
|
106 |
.unwrap()
|
107 |
.value()
|
108 |
.attr("href")
|
109 |
-
.unwrap()
|
110 |
-
.to_string(),
|
111 |
result
|
112 |
.select(&result_desc)
|
113 |
.next()
|
114 |
.unwrap()
|
115 |
.inner_html()
|
116 |
-
.trim()
|
117 |
-
|
118 |
-
vec!["searx".to_string()],
|
119 |
)
|
120 |
})
|
121 |
.map(|search_result| (search_result.url.clone(), search_result))
|
|
|
2 |
//! by querying the upstream searx search engine instance with user provided query and with a page
|
3 |
//! number if provided.
|
4 |
|
5 |
+
use reqwest::header::HeaderMap;
|
6 |
use scraper::{Html, Selector};
|
7 |
use std::collections::HashMap;
|
8 |
|
9 |
use crate::results::aggregation_models::SearchResult;
|
10 |
|
11 |
use super::engine_models::{EngineError, SearchEngine};
|
12 |
+
use error_stack::{Report, Result, ResultExt};
|
13 |
|
14 |
/// A new Searx engine type defined in-order to implement the `SearchEngine` trait which allows to
|
15 |
/// reduce code duplication as well as allows to create vector of different search engines easily.
|
|
|
19 |
impl SearchEngine for Searx {
|
20 |
async fn results(
|
21 |
&self,
|
22 |
+
query: &str,
|
23 |
page: u32,
|
24 |
+
user_agent: &str,
|
25 |
request_timeout: u8,
|
26 |
+
mut safe_search: u8,
|
27 |
) -> Result<HashMap<String, SearchResult>, EngineError> {
|
28 |
// Page number can be missing or empty string and so appropriate handling is required
|
29 |
// so that upstream server recieves valid page number.
|
30 |
+
if safe_search == 3 {
|
31 |
+
safe_search = 2;
|
32 |
+
};
|
33 |
+
|
34 |
let url: String = match page {
|
35 |
+
0 | 1 => {
|
36 |
+
format!("https://searx.work/search?q={query}&pageno=1&safesearch={safe_search}")
|
37 |
+
}
|
38 |
+
_ => format!(
|
39 |
+
"https://searx.work/search?q={query}&pageno={page}&safesearch={safe_search}"
|
40 |
+
),
|
41 |
};
|
42 |
|
43 |
// initializing headers and adding appropriate headers.
|
44 |
+
let header_map = HeaderMap::try_from(&HashMap::from([
|
45 |
+
("USER_AGENT".to_string(), user_agent.to_string()),
|
46 |
+
("REFERER".to_string(), "https://google.com/".to_string()),
|
47 |
+
("CONTENT_TYPE".to_string(), "application/x-www-form-urlencoded".to_string()),
|
48 |
+
("COOKIE".to_string(), "categories=general; language=auto; locale=en; autocomplete=duckduckgo; image_proxy=1; method=POST; safesearch=2; theme=simple; results_on_new_tab=1; doi_resolver=oadoi.org; simple_style=auto; center_alignment=1; query_in_title=1; infinite_scroll=0; disabled_engines=; enabled_engines=\"archive is__general\\054yep__general\\054curlie__general\\054currency__general\\054ddg definitions__general\\054wikidata__general\\054duckduckgo__general\\054tineye__general\\054lingva__general\\054startpage__general\\054yahoo__general\\054wiby__general\\054marginalia__general\\054alexandria__general\\054wikibooks__general\\054wikiquote__general\\054wikisource__general\\054wikiversity__general\\054wikivoyage__general\\054dictzone__general\\054seznam__general\\054mojeek__general\\054naver__general\\054wikimini__general\\054brave__general\\054petalsearch__general\\054goo__general\"; disabled_plugins=; enabled_plugins=\"searx.plugins.hostname_replace\\054searx.plugins.oa_doi_rewrite\\054searx.plugins.vim_hotkeys\"; tokens=; maintab=on; enginetab=on".to_string())
|
49 |
+
]))
|
50 |
+
.change_context(EngineError::UnexpectedError)?;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
51 |
|
52 |
let document: Html = Html::parse_document(
|
53 |
+
&Searx::fetch_html_from_upstream(self, &url, header_map, request_timeout).await?,
|
54 |
);
|
55 |
|
56 |
let no_result: Selector = Selector::parse("#urls>.dialog-error>p")
|
|
|
91 |
.next()
|
92 |
.unwrap()
|
93 |
.inner_html()
|
94 |
+
.trim(),
|
|
|
95 |
result
|
96 |
.select(&result_url)
|
97 |
.next()
|
98 |
.unwrap()
|
99 |
.value()
|
100 |
.attr("href")
|
101 |
+
.unwrap(),
|
|
|
102 |
result
|
103 |
.select(&result_desc)
|
104 |
.next()
|
105 |
.unwrap()
|
106 |
.inner_html()
|
107 |
+
.trim(),
|
108 |
+
&["searx"],
|
|
|
109 |
)
|
110 |
})
|
111 |
.map(|search_result| (search_result.url.clone(), search_result))
|
src/handler/paths.rs
CHANGED
@@ -4,6 +4,7 @@
|
|
4 |
use std::collections::HashMap;
|
5 |
use std::io::Error;
|
6 |
use std::path::Path;
|
|
|
7 |
|
8 |
// ------- Constants --------
|
9 |
/// The constant holding the name of the theme folder.
|
@@ -31,57 +32,7 @@ pub enum FileType {
|
|
31 |
}
|
32 |
|
33 |
/// A static variable which stores the different filesystem paths for various file/folder types.
|
34 |
-
static FILE_PATHS_FOR_DIFF_FILE_TYPES:
|
35 |
-
once_cell::sync::Lazy::new(|| {
|
36 |
-
HashMap::from([
|
37 |
-
(
|
38 |
-
FileType::Config,
|
39 |
-
vec![
|
40 |
-
format!(
|
41 |
-
"{}/.config/{}/{}",
|
42 |
-
std::env::var("HOME").unwrap(),
|
43 |
-
COMMON_DIRECTORY_NAME,
|
44 |
-
CONFIG_FILE_NAME
|
45 |
-
),
|
46 |
-
format!("/etc/xdg/{}/{}", COMMON_DIRECTORY_NAME, CONFIG_FILE_NAME),
|
47 |
-
format!("./{}/{}", COMMON_DIRECTORY_NAME, CONFIG_FILE_NAME),
|
48 |
-
],
|
49 |
-
),
|
50 |
-
(
|
51 |
-
FileType::Theme,
|
52 |
-
vec![
|
53 |
-
format!("/opt/websurfx/{}/", PUBLIC_DIRECTORY_NAME),
|
54 |
-
format!("./{}/", PUBLIC_DIRECTORY_NAME),
|
55 |
-
],
|
56 |
-
),
|
57 |
-
(
|
58 |
-
FileType::AllowList,
|
59 |
-
vec![
|
60 |
-
format!(
|
61 |
-
"{}/.config/{}/{}",
|
62 |
-
std::env::var("HOME").unwrap(),
|
63 |
-
COMMON_DIRECTORY_NAME,
|
64 |
-
ALLOWLIST_FILE_NAME
|
65 |
-
),
|
66 |
-
format!("/etc/xdg/{}/{}", COMMON_DIRECTORY_NAME, ALLOWLIST_FILE_NAME),
|
67 |
-
format!("./{}/{}", COMMON_DIRECTORY_NAME, ALLOWLIST_FILE_NAME),
|
68 |
-
],
|
69 |
-
),
|
70 |
-
(
|
71 |
-
FileType::BlockList,
|
72 |
-
vec![
|
73 |
-
format!(
|
74 |
-
"{}/.config/{}/{}",
|
75 |
-
std::env::var("HOME").unwrap(),
|
76 |
-
COMMON_DIRECTORY_NAME,
|
77 |
-
BLOCKLIST_FILE_NAME
|
78 |
-
),
|
79 |
-
format!("/etc/xdg/{}/{}", COMMON_DIRECTORY_NAME, BLOCKLIST_FILE_NAME),
|
80 |
-
format!("./{}/{}", COMMON_DIRECTORY_NAME, BLOCKLIST_FILE_NAME),
|
81 |
-
],
|
82 |
-
),
|
83 |
-
])
|
84 |
-
});
|
85 |
|
86 |
/// A function which returns an appropriate path for thr provided file type by checking if the path
|
87 |
/// for the given file type exists on that path.
|
@@ -99,11 +50,64 @@ static FILE_PATHS_FOR_DIFF_FILE_TYPES: once_cell::sync::Lazy<HashMap<FileType, V
|
|
99 |
/// 1. `/opt/websurfx` if it not present here then it fallbacks to the next one (2)
|
100 |
/// 2. Under project folder ( or codebase in other words) if it is not present
|
101 |
/// here then it returns an error as mentioned above.
|
102 |
-
pub fn file_path(file_type: FileType) -> Result
|
103 |
-
let file_path = FILE_PATHS_FOR_DIFF_FILE_TYPES
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
104 |
for (idx, _) in file_path.iter().enumerate() {
|
105 |
if Path::new(file_path[idx].as_str()).exists() {
|
106 |
-
return Ok(file_path[idx]
|
107 |
}
|
108 |
}
|
109 |
|
|
|
4 |
use std::collections::HashMap;
|
5 |
use std::io::Error;
|
6 |
use std::path::Path;
|
7 |
+
use std::sync::OnceLock;
|
8 |
|
9 |
// ------- Constants --------
|
10 |
/// The constant holding the name of the theme folder.
|
|
|
32 |
}
|
33 |
|
34 |
/// A static variable which stores the different filesystem paths for various file/folder types.
|
35 |
+
static FILE_PATHS_FOR_DIFF_FILE_TYPES: OnceLock<HashMap<FileType, Vec<String>>> = OnceLock::new();
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
36 |
|
37 |
/// A function which returns an appropriate path for thr provided file type by checking if the path
|
38 |
/// for the given file type exists on that path.
|
|
|
50 |
/// 1. `/opt/websurfx` if it not present here then it fallbacks to the next one (2)
|
51 |
/// 2. Under project folder ( or codebase in other words) if it is not present
|
52 |
/// here then it returns an error as mentioned above.
|
53 |
+
pub fn file_path(file_type: FileType) -> Result<&'static str, Error> {
|
54 |
+
let file_path: &Vec<String> = FILE_PATHS_FOR_DIFF_FILE_TYPES
|
55 |
+
.get_or_init(|| {
|
56 |
+
HashMap::from([
|
57 |
+
(
|
58 |
+
FileType::Config,
|
59 |
+
vec![
|
60 |
+
format!(
|
61 |
+
"{}/.config/{}/{}",
|
62 |
+
std::env::var("HOME").unwrap(),
|
63 |
+
COMMON_DIRECTORY_NAME,
|
64 |
+
CONFIG_FILE_NAME
|
65 |
+
),
|
66 |
+
format!("/etc/xdg/{}/{}", COMMON_DIRECTORY_NAME, CONFIG_FILE_NAME),
|
67 |
+
format!("./{}/{}", COMMON_DIRECTORY_NAME, CONFIG_FILE_NAME),
|
68 |
+
],
|
69 |
+
),
|
70 |
+
(
|
71 |
+
FileType::Theme,
|
72 |
+
vec![
|
73 |
+
format!("/opt/websurfx/{}/", PUBLIC_DIRECTORY_NAME),
|
74 |
+
format!("./{}/", PUBLIC_DIRECTORY_NAME),
|
75 |
+
],
|
76 |
+
),
|
77 |
+
(
|
78 |
+
FileType::AllowList,
|
79 |
+
vec![
|
80 |
+
format!(
|
81 |
+
"{}/.config/{}/{}",
|
82 |
+
std::env::var("HOME").unwrap(),
|
83 |
+
COMMON_DIRECTORY_NAME,
|
84 |
+
ALLOWLIST_FILE_NAME
|
85 |
+
),
|
86 |
+
format!("/etc/xdg/{}/{}", COMMON_DIRECTORY_NAME, ALLOWLIST_FILE_NAME),
|
87 |
+
format!("./{}/{}", COMMON_DIRECTORY_NAME, ALLOWLIST_FILE_NAME),
|
88 |
+
],
|
89 |
+
),
|
90 |
+
(
|
91 |
+
FileType::BlockList,
|
92 |
+
vec![
|
93 |
+
format!(
|
94 |
+
"{}/.config/{}/{}",
|
95 |
+
std::env::var("HOME").unwrap(),
|
96 |
+
COMMON_DIRECTORY_NAME,
|
97 |
+
BLOCKLIST_FILE_NAME
|
98 |
+
),
|
99 |
+
format!("/etc/xdg/{}/{}", COMMON_DIRECTORY_NAME, BLOCKLIST_FILE_NAME),
|
100 |
+
format!("./{}/{}", COMMON_DIRECTORY_NAME, BLOCKLIST_FILE_NAME),
|
101 |
+
],
|
102 |
+
),
|
103 |
+
])
|
104 |
+
})
|
105 |
+
.get(&file_type)
|
106 |
+
.unwrap();
|
107 |
+
|
108 |
for (idx, _) in file_path.iter().enumerate() {
|
109 |
if Path::new(file_path[idx].as_str()).exists() {
|
110 |
+
return Ok(std::mem::take(&mut &*file_path[idx]));
|
111 |
}
|
112 |
}
|
113 |
|
src/lib.rs
CHANGED
@@ -18,6 +18,7 @@ use crate::server::routes;
|
|
18 |
|
19 |
use actix_cors::Cors;
|
20 |
use actix_files as fs;
|
|
|
21 |
use actix_web::{dev::Server, http::header, middleware::Logger, web, App, HttpServer};
|
22 |
use config::parser::Config;
|
23 |
use handlebars::Handlebars;
|
@@ -46,7 +47,7 @@ use handler::paths::{file_path, FileType};
|
|
46 |
pub fn run(listener: TcpListener, config: Config) -> std::io::Result<Server> {
|
47 |
let mut handlebars: Handlebars<'_> = Handlebars::new();
|
48 |
|
49 |
-
let public_folder_path:
|
50 |
|
51 |
handlebars
|
52 |
.register_templates_directory(".html", format!("{}/templates", public_folder_path))
|
@@ -68,10 +69,17 @@ pub fn run(listener: TcpListener, config: Config) -> std::io::Result<Server> {
|
|
68 |
]);
|
69 |
|
70 |
App::new()
|
|
|
71 |
.app_data(handlebars_ref.clone())
|
72 |
.app_data(web::Data::new(config.clone()))
|
73 |
.wrap(cors)
|
74 |
-
.wrap(
|
|
|
|
|
|
|
|
|
|
|
|
|
75 |
// Serve images and static files (css and js files).
|
76 |
.service(
|
77 |
fs::Files::new("/static", format!("{}/static", public_folder_path))
|
|
|
18 |
|
19 |
use actix_cors::Cors;
|
20 |
use actix_files as fs;
|
21 |
+
use actix_governor::{Governor, GovernorConfigBuilder};
|
22 |
use actix_web::{dev::Server, http::header, middleware::Logger, web, App, HttpServer};
|
23 |
use config::parser::Config;
|
24 |
use handlebars::Handlebars;
|
|
|
47 |
pub fn run(listener: TcpListener, config: Config) -> std::io::Result<Server> {
|
48 |
let mut handlebars: Handlebars<'_> = Handlebars::new();
|
49 |
|
50 |
+
let public_folder_path: &str = file_path(FileType::Theme)?;
|
51 |
|
52 |
handlebars
|
53 |
.register_templates_directory(".html", format!("{}/templates", public_folder_path))
|
|
|
69 |
]);
|
70 |
|
71 |
App::new()
|
72 |
+
.wrap(Logger::default()) // added logging middleware for logging.
|
73 |
.app_data(handlebars_ref.clone())
|
74 |
.app_data(web::Data::new(config.clone()))
|
75 |
.wrap(cors)
|
76 |
+
.wrap(Governor::new(
|
77 |
+
&GovernorConfigBuilder::default()
|
78 |
+
.per_second(config.rate_limiter.time_limit as u64)
|
79 |
+
.burst_size(config.rate_limiter.number_of_requests as u32)
|
80 |
+
.finish()
|
81 |
+
.unwrap(),
|
82 |
+
))
|
83 |
// Serve images and static files (css and js files).
|
84 |
.service(
|
85 |
fs::Files::new("/static", format!("{}/static", public_folder_path))
|
src/results/aggregation_models.rs
CHANGED
@@ -2,6 +2,7 @@
|
|
2 |
//! data scraped from the upstream search engines.
|
3 |
|
4 |
use serde::{Deserialize, Serialize};
|
|
|
5 |
|
6 |
use crate::{config::parser_models::Style, engines::engine_models::EngineError};
|
7 |
|
@@ -19,7 +20,7 @@ pub struct SearchResult {
|
|
19 |
/// The description of the search result.
|
20 |
pub description: String,
|
21 |
/// The names of the upstream engines from which this results were provided.
|
22 |
-
pub engine:
|
23 |
}
|
24 |
|
25 |
impl SearchResult {
|
@@ -32,12 +33,12 @@ impl SearchResult {
|
|
32 |
/// (href url in html in simple words).
|
33 |
/// * `description` - The description of the search result.
|
34 |
/// * `engine` - The names of the upstream engines from which this results were provided.
|
35 |
-
pub fn new(title:
|
36 |
SearchResult {
|
37 |
-
title,
|
38 |
-
url,
|
39 |
-
description,
|
40 |
-
engine,
|
41 |
}
|
42 |
}
|
43 |
|
@@ -46,8 +47,8 @@ impl SearchResult {
|
|
46 |
/// # Arguments
|
47 |
///
|
48 |
/// * `engine` - Takes an engine name provided as a String.
|
49 |
-
pub fn add_engines(&mut self, engine:
|
50 |
-
self.engine.push(engine)
|
51 |
}
|
52 |
|
53 |
/// A function which returns the engine name stored from the struct as a string.
|
@@ -55,13 +56,13 @@ impl SearchResult {
|
|
55 |
/// # Returns
|
56 |
///
|
57 |
/// An engine name stored as a string from the struct.
|
58 |
-
pub fn engine(self) -> String {
|
59 |
-
self.engine
|
60 |
}
|
61 |
}
|
62 |
|
63 |
/// A named struct that stores the error info related to the upstream search engines.
|
64 |
-
#[derive(Serialize, Deserialize)]
|
65 |
pub struct EngineErrorInfo {
|
66 |
/// It stores the error type which occured while fetching the result from a particular search
|
67 |
/// engine.
|
@@ -81,18 +82,18 @@ impl EngineErrorInfo {
|
|
81 |
/// * `error` - It takes the error type which occured while fetching the result from a particular
|
82 |
/// search engine.
|
83 |
/// * `engine` - It takes the name of the engine that failed to provide the requested search results.
|
84 |
-
pub fn new(error: &EngineError, engine:
|
85 |
Self {
|
86 |
error: match error {
|
87 |
-
EngineError::RequestError =>
|
88 |
-
EngineError::EmptyResultSet =>
|
89 |
-
EngineError::UnexpectedError =>
|
90 |
},
|
91 |
-
engine,
|
92 |
severity_color: match error {
|
93 |
-
EngineError::RequestError =>
|
94 |
-
EngineError::EmptyResultSet =>
|
95 |
-
EngineError::UnexpectedError =>
|
96 |
},
|
97 |
}
|
98 |
}
|
@@ -101,7 +102,7 @@ impl EngineErrorInfo {
|
|
101 |
/// A named struct to store, serialize, deserialize the all the search results scraped and
|
102 |
/// aggregated from the upstream search engines.
|
103 |
/// `SearchResult` structs.
|
104 |
-
#[derive(Serialize, Deserialize)]
|
105 |
#[serde(rename_all = "camelCase")]
|
106 |
pub struct SearchResults {
|
107 |
/// Stores the individual serializable `SearchResult` struct into a vector of
|
@@ -113,6 +114,14 @@ pub struct SearchResults {
|
|
113 |
/// Stores the information on which engines failed with their engine name
|
114 |
/// and the type of error that caused it.
|
115 |
pub engine_errors_info: Vec<EngineErrorInfo>,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
116 |
}
|
117 |
|
118 |
impl SearchResults {
|
@@ -126,21 +135,48 @@ impl SearchResults {
|
|
126 |
/// the search url.
|
127 |
/// * `empty_result_set` - Takes a boolean which indicates that no engines gave a result for the
|
128 |
/// given search query.
|
|
|
129 |
pub fn new(
|
130 |
results: Vec<SearchResult>,
|
131 |
-
page_query:
|
132 |
-
engine_errors_info:
|
133 |
) -> Self {
|
134 |
-
|
135 |
results,
|
136 |
-
page_query,
|
137 |
-
style: Style::
|
138 |
-
engine_errors_info,
|
|
|
|
|
139 |
}
|
140 |
}
|
141 |
|
142 |
/// A setter function to add website style to the return search results.
|
143 |
-
pub fn add_style(&mut self, style: Style) {
|
144 |
-
self.style = style;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
145 |
}
|
146 |
}
|
|
|
2 |
//! data scraped from the upstream search engines.
|
3 |
|
4 |
use serde::{Deserialize, Serialize};
|
5 |
+
use smallvec::SmallVec;
|
6 |
|
7 |
use crate::{config::parser_models::Style, engines::engine_models::EngineError};
|
8 |
|
|
|
20 |
/// The description of the search result.
|
21 |
pub description: String,
|
22 |
/// The names of the upstream engines from which this results were provided.
|
23 |
+
pub engine: SmallVec<[String; 0]>,
|
24 |
}
|
25 |
|
26 |
impl SearchResult {
|
|
|
33 |
/// (href url in html in simple words).
|
34 |
/// * `description` - The description of the search result.
|
35 |
/// * `engine` - The names of the upstream engines from which this results were provided.
|
36 |
+
pub fn new(title: &str, url: &str, description: &str, engine: &[&str]) -> Self {
|
37 |
SearchResult {
|
38 |
+
title: title.to_owned(),
|
39 |
+
url: url.to_owned(),
|
40 |
+
description: description.to_owned(),
|
41 |
+
engine: engine.iter().map(|name| name.to_string()).collect(),
|
42 |
}
|
43 |
}
|
44 |
|
|
|
47 |
/// # Arguments
|
48 |
///
|
49 |
/// * `engine` - Takes an engine name provided as a String.
|
50 |
+
pub fn add_engines(&mut self, engine: &str) {
|
51 |
+
self.engine.push(engine.to_owned())
|
52 |
}
|
53 |
|
54 |
/// A function which returns the engine name stored from the struct as a string.
|
|
|
56 |
/// # Returns
|
57 |
///
|
58 |
/// An engine name stored as a string from the struct.
|
59 |
+
pub fn engine(&mut self) -> String {
|
60 |
+
std::mem::take(&mut self.engine[0])
|
61 |
}
|
62 |
}
|
63 |
|
64 |
/// A named struct that stores the error info related to the upstream search engines.
|
65 |
+
#[derive(Serialize, Deserialize, Clone)]
|
66 |
pub struct EngineErrorInfo {
|
67 |
/// It stores the error type which occured while fetching the result from a particular search
|
68 |
/// engine.
|
|
|
82 |
/// * `error` - It takes the error type which occured while fetching the result from a particular
|
83 |
/// search engine.
|
84 |
/// * `engine` - It takes the name of the engine that failed to provide the requested search results.
|
85 |
+
pub fn new(error: &EngineError, engine: &str) -> Self {
|
86 |
Self {
|
87 |
error: match error {
|
88 |
+
EngineError::RequestError => "RequestError".to_owned(),
|
89 |
+
EngineError::EmptyResultSet => "EmptyResultSet".to_owned(),
|
90 |
+
EngineError::UnexpectedError => "UnexpectedError".to_owned(),
|
91 |
},
|
92 |
+
engine: engine.to_owned(),
|
93 |
severity_color: match error {
|
94 |
+
EngineError::RequestError => "green".to_owned(),
|
95 |
+
EngineError::EmptyResultSet => "blue".to_owned(),
|
96 |
+
EngineError::UnexpectedError => "red".to_owned(),
|
97 |
},
|
98 |
}
|
99 |
}
|
|
|
102 |
/// A named struct to store, serialize, deserialize the all the search results scraped and
|
103 |
/// aggregated from the upstream search engines.
|
104 |
/// `SearchResult` structs.
|
105 |
+
#[derive(Serialize, Deserialize, Default)]
|
106 |
#[serde(rename_all = "camelCase")]
|
107 |
pub struct SearchResults {
|
108 |
/// Stores the individual serializable `SearchResult` struct into a vector of
|
|
|
114 |
/// Stores the information on which engines failed with their engine name
|
115 |
/// and the type of error that caused it.
|
116 |
pub engine_errors_info: Vec<EngineErrorInfo>,
|
117 |
+
/// Stores the flag option which holds the check value that the following
|
118 |
+
/// search query was disallowed when the safe search level set to 4 and it
|
119 |
+
/// was present in the `Blocklist` file.
|
120 |
+
pub disallowed: bool,
|
121 |
+
/// Stores the flag option which holds the check value that the following
|
122 |
+
/// search query was filtered when the safe search level set to 3 and it
|
123 |
+
/// was present in the `Blocklist` file.
|
124 |
+
pub filtered: bool,
|
125 |
}
|
126 |
|
127 |
impl SearchResults {
|
|
|
135 |
/// the search url.
|
136 |
/// * `empty_result_set` - Takes a boolean which indicates that no engines gave a result for the
|
137 |
/// given search query.
|
138 |
+
/// * ``
|
139 |
pub fn new(
|
140 |
results: Vec<SearchResult>,
|
141 |
+
page_query: &str,
|
142 |
+
engine_errors_info: &[EngineErrorInfo],
|
143 |
) -> Self {
|
144 |
+
Self {
|
145 |
results,
|
146 |
+
page_query: page_query.to_owned(),
|
147 |
+
style: Style::default(),
|
148 |
+
engine_errors_info: engine_errors_info.to_owned(),
|
149 |
+
disallowed: Default::default(),
|
150 |
+
filtered: Default::default(),
|
151 |
}
|
152 |
}
|
153 |
|
154 |
/// A setter function to add website style to the return search results.
|
155 |
+
pub fn add_style(&mut self, style: &Style) {
|
156 |
+
self.style = style.clone();
|
157 |
+
}
|
158 |
+
|
159 |
+
/// A setter function that sets disallowed to true.
|
160 |
+
pub fn set_disallowed(&mut self) {
|
161 |
+
self.disallowed = true;
|
162 |
+
}
|
163 |
+
|
164 |
+
/// A setter function to set the current page search query.
|
165 |
+
pub fn set_page_query(&mut self, page: &str) {
|
166 |
+
self.page_query = page.to_owned();
|
167 |
+
}
|
168 |
+
|
169 |
+
/// A setter function that sets the filtered to true.
|
170 |
+
pub fn set_filtered(&mut self) {
|
171 |
+
self.filtered = true;
|
172 |
+
}
|
173 |
+
|
174 |
+
/// A getter function that gets the value of `engine_errors_info`.
|
175 |
+
pub fn engine_errors_info(&mut self) -> Vec<EngineErrorInfo> {
|
176 |
+
std::mem::take(&mut self.engine_errors_info)
|
177 |
+
}
|
178 |
+
/// A getter function that gets the value of `results`.
|
179 |
+
pub fn results(&mut self) -> Vec<SearchResult> {
|
180 |
+
self.results.clone()
|
181 |
}
|
182 |
}
|
src/results/aggregator.rs
CHANGED
@@ -64,14 +64,15 @@ type FutureVec = Vec<JoinHandle<Result<HashMap<String, SearchResult>, Report<Eng
|
|
64 |
/// function in either `searx` or `duckduckgo` or both otherwise returns a `SearchResults struct`
|
65 |
/// containing appropriate values.
|
66 |
pub async fn aggregate(
|
67 |
-
query:
|
68 |
page: u32,
|
69 |
random_delay: bool,
|
70 |
debug: bool,
|
71 |
-
upstream_search_engines:
|
72 |
request_timeout: u8,
|
|
|
73 |
) -> Result<SearchResults, Box<dyn std::error::Error>> {
|
74 |
-
let user_agent:
|
75 |
|
76 |
// Add a random delay before making the request.
|
77 |
if random_delay || !debug {
|
@@ -80,19 +81,24 @@ pub async fn aggregate(
|
|
80 |
tokio::time::sleep(Duration::from_secs(delay_secs)).await;
|
81 |
}
|
82 |
|
83 |
-
let mut names: Vec<&str> =
|
84 |
|
85 |
// create tasks for upstream result fetching
|
86 |
let mut tasks: FutureVec = FutureVec::new();
|
87 |
|
88 |
for engine_handler in upstream_search_engines {
|
89 |
-
let (name, search_engine) = engine_handler.into_name_engine();
|
90 |
names.push(name);
|
91 |
-
let query: String = query.
|
92 |
-
let user_agent: String = user_agent.clone();
|
93 |
tasks.push(tokio::spawn(async move {
|
94 |
search_engine
|
95 |
-
.results(
|
|
|
|
|
|
|
|
|
|
|
|
|
96 |
.await
|
97 |
}));
|
98 |
}
|
@@ -110,7 +116,7 @@ pub async fn aggregate(
|
|
110 |
let mut result_map: HashMap<String, SearchResult> = HashMap::new();
|
111 |
let mut engine_errors_info: Vec<EngineErrorInfo> = Vec::new();
|
112 |
|
113 |
-
let mut handle_error = |error: Report<EngineError>, engine_name:
|
114 |
log::error!("Engine Error: {:?}", error);
|
115 |
engine_errors_info.push(EngineErrorInfo::new(
|
116 |
error.downcast_ref::<EngineError>().unwrap(),
|
@@ -120,7 +126,7 @@ pub async fn aggregate(
|
|
120 |
|
121 |
for _ in 0..responses.len() {
|
122 |
let response = responses.pop().unwrap();
|
123 |
-
let engine = names.pop().unwrap()
|
124 |
|
125 |
if result_map.is_empty() {
|
126 |
match response {
|
@@ -128,7 +134,7 @@ pub async fn aggregate(
|
|
128 |
result_map = results.clone();
|
129 |
}
|
130 |
Err(error) => {
|
131 |
-
handle_error(error, engine);
|
132 |
}
|
133 |
}
|
134 |
continue;
|
@@ -140,39 +146,37 @@ pub async fn aggregate(
|
|
140 |
result_map
|
141 |
.entry(key)
|
142 |
.and_modify(|result| {
|
143 |
-
result.add_engines(engine
|
144 |
})
|
145 |
.or_insert_with(|| -> SearchResult { value });
|
146 |
});
|
147 |
}
|
148 |
Err(error) => {
|
149 |
-
handle_error(error, engine);
|
150 |
}
|
151 |
}
|
152 |
}
|
153 |
|
154 |
-
|
155 |
-
|
156 |
-
|
157 |
-
|
158 |
-
|
159 |
-
|
|
|
160 |
|
161 |
-
|
162 |
-
|
163 |
-
|
164 |
-
|
165 |
-
|
166 |
|
167 |
-
|
|
|
168 |
|
169 |
let results: Vec<SearchResult> = result_map.into_values().collect();
|
170 |
|
171 |
-
Ok(SearchResults::new(
|
172 |
-
results,
|
173 |
-
query.to_string(),
|
174 |
-
engine_errors_info,
|
175 |
-
))
|
176 |
}
|
177 |
|
178 |
/// Filters a map of search results using a list of regex patterns.
|
@@ -194,7 +198,7 @@ pub fn filter_with_lists(
|
|
194 |
let mut reader = BufReader::new(File::open(file_path)?);
|
195 |
|
196 |
for line in reader.by_ref().lines() {
|
197 |
-
let re = Regex::new(
|
198 |
|
199 |
// Iterate over each search result in the map and check if it matches the regex pattern
|
200 |
for (url, search_result) in map_to_be_filtered.clone().into_iter() {
|
@@ -203,7 +207,10 @@ pub fn filter_with_lists(
|
|
203 |
|| re.is_match(&search_result.description.to_lowercase())
|
204 |
{
|
205 |
// If the search result matches the regex pattern, move it from the original map to the resultant map
|
206 |
-
resultant_map.insert(
|
|
|
|
|
|
|
207 |
}
|
208 |
}
|
209 |
}
|
@@ -214,6 +221,7 @@ pub fn filter_with_lists(
|
|
214 |
#[cfg(test)]
|
215 |
mod tests {
|
216 |
use super::*;
|
|
|
217 |
use std::collections::HashMap;
|
218 |
use std::io::Write;
|
219 |
use tempfile::NamedTempFile;
|
@@ -223,22 +231,22 @@ mod tests {
|
|
223 |
// Create a map of search results to filter
|
224 |
let mut map_to_be_filtered = HashMap::new();
|
225 |
map_to_be_filtered.insert(
|
226 |
-
"https://www.example.com".
|
227 |
SearchResult {
|
228 |
-
title: "Example Domain".
|
229 |
-
url: "https://www.example.com".
|
230 |
description: "This domain is for use in illustrative examples in documents."
|
231 |
-
.
|
232 |
-
engine:
|
233 |
},
|
234 |
);
|
235 |
map_to_be_filtered.insert(
|
236 |
-
"https://www.rust-lang.org/".
|
237 |
SearchResult {
|
238 |
-
title: "Rust Programming Language".
|
239 |
-
url: "https://www.rust-lang.org/".
|
240 |
-
description: "A systems programming language that runs blazingly fast, prevents segfaults, and guarantees thread safety.".
|
241 |
-
engine:
|
242 |
},
|
243 |
);
|
244 |
|
@@ -267,22 +275,22 @@ mod tests {
|
|
267 |
fn test_filter_with_lists_wildcard() -> Result<(), Box<dyn std::error::Error>> {
|
268 |
let mut map_to_be_filtered = HashMap::new();
|
269 |
map_to_be_filtered.insert(
|
270 |
-
"https://www.example.com".
|
271 |
SearchResult {
|
272 |
-
title: "Example Domain".
|
273 |
-
url: "https://www.example.com".
|
274 |
description: "This domain is for use in illustrative examples in documents."
|
275 |
-
.
|
276 |
-
engine:
|
277 |
},
|
278 |
);
|
279 |
map_to_be_filtered.insert(
|
280 |
-
"https://www.rust-lang.org/".
|
281 |
SearchResult {
|
282 |
-
title: "Rust Programming Language".
|
283 |
-
url: "https://www.rust-lang.org/".
|
284 |
-
description: "A systems programming language that runs blazingly fast, prevents segfaults, and guarantees thread safety.".
|
285 |
-
engine:
|
286 |
},
|
287 |
);
|
288 |
|
@@ -327,13 +335,13 @@ mod tests {
|
|
327 |
fn test_filter_with_lists_invalid_regex() {
|
328 |
let mut map_to_be_filtered = HashMap::new();
|
329 |
map_to_be_filtered.insert(
|
330 |
-
"https://www.example.com".
|
331 |
SearchResult {
|
332 |
-
title: "Example Domain".
|
333 |
-
url: "https://www.example.com".
|
334 |
description: "This domain is for use in illustrative examples in documents."
|
335 |
-
.
|
336 |
-
engine:
|
337 |
},
|
338 |
);
|
339 |
|
|
|
64 |
/// function in either `searx` or `duckduckgo` or both otherwise returns a `SearchResults struct`
|
65 |
/// containing appropriate values.
|
66 |
pub async fn aggregate(
|
67 |
+
query: &str,
|
68 |
page: u32,
|
69 |
random_delay: bool,
|
70 |
debug: bool,
|
71 |
+
upstream_search_engines: &[EngineHandler],
|
72 |
request_timeout: u8,
|
73 |
+
safe_search: u8,
|
74 |
) -> Result<SearchResults, Box<dyn std::error::Error>> {
|
75 |
+
let user_agent: &str = random_user_agent();
|
76 |
|
77 |
// Add a random delay before making the request.
|
78 |
if random_delay || !debug {
|
|
|
81 |
tokio::time::sleep(Duration::from_secs(delay_secs)).await;
|
82 |
}
|
83 |
|
84 |
+
let mut names: Vec<&str> = Vec::with_capacity(0);
|
85 |
|
86 |
// create tasks for upstream result fetching
|
87 |
let mut tasks: FutureVec = FutureVec::new();
|
88 |
|
89 |
for engine_handler in upstream_search_engines {
|
90 |
+
let (name, search_engine) = engine_handler.to_owned().into_name_engine();
|
91 |
names.push(name);
|
92 |
+
let query: String = query.to_owned();
|
|
|
93 |
tasks.push(tokio::spawn(async move {
|
94 |
search_engine
|
95 |
+
.results(
|
96 |
+
&query,
|
97 |
+
page,
|
98 |
+
user_agent.clone(),
|
99 |
+
request_timeout,
|
100 |
+
safe_search,
|
101 |
+
)
|
102 |
.await
|
103 |
}));
|
104 |
}
|
|
|
116 |
let mut result_map: HashMap<String, SearchResult> = HashMap::new();
|
117 |
let mut engine_errors_info: Vec<EngineErrorInfo> = Vec::new();
|
118 |
|
119 |
+
let mut handle_error = |error: &Report<EngineError>, engine_name: &'static str| {
|
120 |
log::error!("Engine Error: {:?}", error);
|
121 |
engine_errors_info.push(EngineErrorInfo::new(
|
122 |
error.downcast_ref::<EngineError>().unwrap(),
|
|
|
126 |
|
127 |
for _ in 0..responses.len() {
|
128 |
let response = responses.pop().unwrap();
|
129 |
+
let engine = names.pop().unwrap();
|
130 |
|
131 |
if result_map.is_empty() {
|
132 |
match response {
|
|
|
134 |
result_map = results.clone();
|
135 |
}
|
136 |
Err(error) => {
|
137 |
+
handle_error(&error, engine);
|
138 |
}
|
139 |
}
|
140 |
continue;
|
|
|
146 |
result_map
|
147 |
.entry(key)
|
148 |
.and_modify(|result| {
|
149 |
+
result.add_engines(engine);
|
150 |
})
|
151 |
.or_insert_with(|| -> SearchResult { value });
|
152 |
});
|
153 |
}
|
154 |
Err(error) => {
|
155 |
+
handle_error(&error, engine);
|
156 |
}
|
157 |
}
|
158 |
}
|
159 |
|
160 |
+
if safe_search >= 3 {
|
161 |
+
let mut blacklist_map: HashMap<String, SearchResult> = HashMap::new();
|
162 |
+
filter_with_lists(
|
163 |
+
&mut result_map,
|
164 |
+
&mut blacklist_map,
|
165 |
+
file_path(FileType::BlockList)?,
|
166 |
+
)?;
|
167 |
|
168 |
+
filter_with_lists(
|
169 |
+
&mut blacklist_map,
|
170 |
+
&mut result_map,
|
171 |
+
file_path(FileType::AllowList)?,
|
172 |
+
)?;
|
173 |
|
174 |
+
drop(blacklist_map);
|
175 |
+
}
|
176 |
|
177 |
let results: Vec<SearchResult> = result_map.into_values().collect();
|
178 |
|
179 |
+
Ok(SearchResults::new(results, query, &engine_errors_info))
|
|
|
|
|
|
|
|
|
180 |
}
|
181 |
|
182 |
/// Filters a map of search results using a list of regex patterns.
|
|
|
198 |
let mut reader = BufReader::new(File::open(file_path)?);
|
199 |
|
200 |
for line in reader.by_ref().lines() {
|
201 |
+
let re = Regex::new(line?.trim())?;
|
202 |
|
203 |
// Iterate over each search result in the map and check if it matches the regex pattern
|
204 |
for (url, search_result) in map_to_be_filtered.clone().into_iter() {
|
|
|
207 |
|| re.is_match(&search_result.description.to_lowercase())
|
208 |
{
|
209 |
// If the search result matches the regex pattern, move it from the original map to the resultant map
|
210 |
+
resultant_map.insert(
|
211 |
+
url.to_owned(),
|
212 |
+
map_to_be_filtered.remove(&url.to_owned()).unwrap(),
|
213 |
+
);
|
214 |
}
|
215 |
}
|
216 |
}
|
|
|
221 |
#[cfg(test)]
|
222 |
mod tests {
|
223 |
use super::*;
|
224 |
+
use smallvec::smallvec;
|
225 |
use std::collections::HashMap;
|
226 |
use std::io::Write;
|
227 |
use tempfile::NamedTempFile;
|
|
|
231 |
// Create a map of search results to filter
|
232 |
let mut map_to_be_filtered = HashMap::new();
|
233 |
map_to_be_filtered.insert(
|
234 |
+
"https://www.example.com".to_owned(),
|
235 |
SearchResult {
|
236 |
+
title: "Example Domain".to_owned(),
|
237 |
+
url: "https://www.example.com".to_owned(),
|
238 |
description: "This domain is for use in illustrative examples in documents."
|
239 |
+
.to_owned(),
|
240 |
+
engine: smallvec!["Google".to_owned(), "Bing".to_owned()],
|
241 |
},
|
242 |
);
|
243 |
map_to_be_filtered.insert(
|
244 |
+
"https://www.rust-lang.org/".to_owned(),
|
245 |
SearchResult {
|
246 |
+
title: "Rust Programming Language".to_owned(),
|
247 |
+
url: "https://www.rust-lang.org/".to_owned(),
|
248 |
+
description: "A systems programming language that runs blazingly fast, prevents segfaults, and guarantees thread safety.".to_owned(),
|
249 |
+
engine: smallvec!["Google".to_owned(), "DuckDuckGo".to_owned()],
|
250 |
},
|
251 |
);
|
252 |
|
|
|
275 |
fn test_filter_with_lists_wildcard() -> Result<(), Box<dyn std::error::Error>> {
|
276 |
let mut map_to_be_filtered = HashMap::new();
|
277 |
map_to_be_filtered.insert(
|
278 |
+
"https://www.example.com".to_owned(),
|
279 |
SearchResult {
|
280 |
+
title: "Example Domain".to_owned(),
|
281 |
+
url: "https://www.example.com".to_owned(),
|
282 |
description: "This domain is for use in illustrative examples in documents."
|
283 |
+
.to_owned(),
|
284 |
+
engine: smallvec!["Google".to_owned(), "Bing".to_owned()],
|
285 |
},
|
286 |
);
|
287 |
map_to_be_filtered.insert(
|
288 |
+
"https://www.rust-lang.org/".to_owned(),
|
289 |
SearchResult {
|
290 |
+
title: "Rust Programming Language".to_owned(),
|
291 |
+
url: "https://www.rust-lang.org/".to_owned(),
|
292 |
+
description: "A systems programming language that runs blazingly fast, prevents segfaults, and guarantees thread safety.".to_owned(),
|
293 |
+
engine: smallvec!["Google".to_owned(), "DuckDuckGo".to_owned()],
|
294 |
},
|
295 |
);
|
296 |
|
|
|
335 |
fn test_filter_with_lists_invalid_regex() {
|
336 |
let mut map_to_be_filtered = HashMap::new();
|
337 |
map_to_be_filtered.insert(
|
338 |
+
"https://www.example.com".to_owned(),
|
339 |
SearchResult {
|
340 |
+
title: "Example Domain".to_owned(),
|
341 |
+
url: "https://www.example.com".to_owned(),
|
342 |
description: "This domain is for use in illustrative examples in documents."
|
343 |
+
.to_owned(),
|
344 |
+
engine: smallvec!["Google".to_owned(), "Bing".to_owned()],
|
345 |
},
|
346 |
);
|
347 |
|
src/results/user_agent.rs
CHANGED
@@ -1,30 +1,34 @@
|
|
1 |
//! This module provides the functionality to generate random user agent string.
|
2 |
|
|
|
|
|
3 |
use fake_useragent::{Browsers, UserAgents, UserAgentsBuilder};
|
4 |
|
5 |
/// A static variable which stores the initially build `UserAgents` struct. So as it can be resused
|
6 |
/// again and again without the need of reinitializing the `UserAgents` struct.
|
7 |
-
static USER_AGENTS:
|
8 |
-
UserAgentsBuilder::new()
|
9 |
-
.cache(false)
|
10 |
-
.dir("/tmp")
|
11 |
-
.thread(1)
|
12 |
-
.set_browsers(
|
13 |
-
Browsers::new()
|
14 |
-
.set_chrome()
|
15 |
-
.set_safari()
|
16 |
-
.set_edge()
|
17 |
-
.set_firefox()
|
18 |
-
.set_mozilla(),
|
19 |
-
)
|
20 |
-
.build()
|
21 |
-
});
|
22 |
|
23 |
/// A function to generate random user agent to improve privacy of the user.
|
24 |
///
|
25 |
/// # Returns
|
26 |
///
|
27 |
/// A randomly generated user agent string.
|
28 |
-
pub fn random_user_agent() ->
|
29 |
-
USER_AGENTS
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
30 |
}
|
|
|
1 |
//! This module provides the functionality to generate random user agent string.
|
2 |
|
3 |
+
use std::sync::OnceLock;
|
4 |
+
|
5 |
use fake_useragent::{Browsers, UserAgents, UserAgentsBuilder};
|
6 |
|
7 |
/// A static variable which stores the initially build `UserAgents` struct. So as it can be resused
|
8 |
/// again and again without the need of reinitializing the `UserAgents` struct.
|
9 |
+
static USER_AGENTS: OnceLock<UserAgents> = OnceLock::new();
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
10 |
|
11 |
/// A function to generate random user agent to improve privacy of the user.
|
12 |
///
|
13 |
/// # Returns
|
14 |
///
|
15 |
/// A randomly generated user agent string.
|
16 |
+
pub fn random_user_agent() -> &'static str {
|
17 |
+
USER_AGENTS
|
18 |
+
.get_or_init(|| {
|
19 |
+
UserAgentsBuilder::new()
|
20 |
+
.cache(false)
|
21 |
+
.dir("/tmp")
|
22 |
+
.thread(1)
|
23 |
+
.set_browsers(
|
24 |
+
Browsers::new()
|
25 |
+
.set_chrome()
|
26 |
+
.set_safari()
|
27 |
+
.set_edge()
|
28 |
+
.set_firefox()
|
29 |
+
.set_mozilla(),
|
30 |
+
)
|
31 |
+
.build()
|
32 |
+
})
|
33 |
+
.random()
|
34 |
}
|
src/server/routes.rs
CHANGED
@@ -2,7 +2,10 @@
|
|
2 |
//! meta search engine website and provide appropriate response to each route/page
|
3 |
//! when requested.
|
4 |
|
5 |
-
use std::
|
|
|
|
|
|
|
6 |
|
7 |
use crate::{
|
8 |
cache::cacher::RedisCache,
|
@@ -13,9 +16,14 @@ use crate::{
|
|
13 |
};
|
14 |
use actix_web::{get, web, HttpRequest, HttpResponse};
|
15 |
use handlebars::Handlebars;
|
|
|
16 |
use serde::Deserialize;
|
17 |
use tokio::join;
|
18 |
|
|
|
|
|
|
|
|
|
19 |
/// A named struct which deserializes all the user provided search parameters and stores them.
|
20 |
#[derive(Deserialize)]
|
21 |
struct SearchParams {
|
@@ -25,6 +33,7 @@ struct SearchParams {
|
|
25 |
/// It stores the search parameter `page` (or pageno in simple words)
|
26 |
/// of the search url.
|
27 |
page: Option<u32>,
|
|
|
28 |
}
|
29 |
|
30 |
/// Handles the route of index page or main page of the `websurfx` meta search engine website.
|
@@ -53,13 +62,13 @@ pub async fn not_found(
|
|
53 |
/// A named struct which is used to deserialize the cookies fetched from the client side.
|
54 |
#[allow(dead_code)]
|
55 |
#[derive(Deserialize)]
|
56 |
-
struct Cookie {
|
57 |
/// It stores the theme name used in the website.
|
58 |
-
theme:
|
59 |
/// It stores the colorscheme name used for the website theme.
|
60 |
-
colorscheme:
|
61 |
/// It stores the user selected upstream search engines selected from the UI.
|
62 |
-
engines: Vec
|
63 |
}
|
64 |
|
65 |
/// Handles the route of search page of the `websurfx` meta search engine website and it takes
|
@@ -95,42 +104,58 @@ pub async fn search(
|
|
95 |
None => 1,
|
96 |
};
|
97 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
98 |
let (_, results, _) = join!(
|
99 |
results(
|
100 |
format!(
|
101 |
-
"http://{}:{}/search?q={}&page={}",
|
102 |
config.binding_ip,
|
103 |
config.port,
|
104 |
query,
|
105 |
-
page - 1
|
|
|
106 |
),
|
107 |
&config,
|
108 |
-
query
|
109 |
page - 1,
|
110 |
req.clone(),
|
|
|
111 |
),
|
112 |
results(
|
113 |
format!(
|
114 |
-
"http://{}:{}/search?q={}&page={}",
|
115 |
-
config.binding_ip, config.port, query, page
|
116 |
),
|
117 |
&config,
|
118 |
-
query
|
119 |
page,
|
120 |
req.clone(),
|
|
|
121 |
),
|
122 |
results(
|
123 |
format!(
|
124 |
-
"http://{}:{}/search?q={}&page={}",
|
125 |
config.binding_ip,
|
126 |
config.port,
|
127 |
query,
|
128 |
-
page + 1
|
|
|
129 |
),
|
130 |
&config,
|
131 |
-
query
|
132 |
page + 1,
|
133 |
req.clone(),
|
|
|
134 |
)
|
135 |
);
|
136 |
|
@@ -161,30 +186,53 @@ pub async fn search(
|
|
161 |
async fn results(
|
162 |
url: String,
|
163 |
config: &Config,
|
164 |
-
query:
|
165 |
page: u32,
|
166 |
req: HttpRequest,
|
|
|
167 |
) -> Result<SearchResults, Box<dyn std::error::Error>> {
|
168 |
// Initialize redis cache connection struct
|
169 |
-
let mut redis_cache =
|
|
|
|
|
|
|
|
|
|
|
|
|
170 |
// fetch the cached results json.
|
171 |
-
let cached_results_json =
|
|
|
172 |
// check if fetched cache results was indeed fetched or it was an error and if so
|
173 |
// handle the data accordingly.
|
174 |
match cached_results_json {
|
175 |
-
Ok(results) => Ok(serde_json::from_str::<SearchResults>(&results)
|
176 |
Err(_) => {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
177 |
// check if the cookie value is empty or not if it is empty then use the
|
178 |
// default selected upstream search engines from the config file otherwise
|
179 |
// parse the non-empty cookie and grab the user selected engines from the
|
180 |
// UI and use that.
|
181 |
-
let mut results:
|
182 |
-
.cookie("appCookie")
|
183 |
-
{
|
184 |
Some(cookie_value) => {
|
185 |
let cookie_value: Cookie = serde_json::from_str(cookie_value.name_value().1)?;
|
186 |
|
187 |
-
let engines = cookie_value
|
188 |
.engines
|
189 |
.iter()
|
190 |
.filter_map(|name| EngineHandler::new(name))
|
@@ -195,8 +243,9 @@ async fn results(
|
|
195 |
page,
|
196 |
config.aggregator.random_delay,
|
197 |
config.debug,
|
198 |
-
engines,
|
199 |
config.request_timeout,
|
|
|
200 |
)
|
201 |
.await?
|
202 |
}
|
@@ -206,19 +255,41 @@ async fn results(
|
|
206 |
page,
|
207 |
config.aggregator.random_delay,
|
208 |
config.debug,
|
209 |
-
config.upstream_search_engines
|
210 |
config.request_timeout,
|
|
|
211 |
)
|
212 |
.await?
|
213 |
}
|
214 |
};
|
215 |
-
results.
|
216 |
-
|
|
|
|
|
|
|
|
|
|
|
217 |
Ok(results)
|
218 |
}
|
219 |
}
|
220 |
}
|
221 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
222 |
/// Handles the route of robots.txt page of the `websurfx` meta search engine website.
|
223 |
#[get("/robots.txt")]
|
224 |
pub async fn robots_data(_req: HttpRequest) -> Result<HttpResponse, Box<dyn std::error::Error>> {
|
|
|
2 |
//! meta search engine website and provide appropriate response to each route/page
|
3 |
//! when requested.
|
4 |
|
5 |
+
use std::{
|
6 |
+
fs::{read_to_string, File},
|
7 |
+
io::{BufRead, BufReader, Read},
|
8 |
+
};
|
9 |
|
10 |
use crate::{
|
11 |
cache::cacher::RedisCache,
|
|
|
16 |
};
|
17 |
use actix_web::{get, web, HttpRequest, HttpResponse};
|
18 |
use handlebars::Handlebars;
|
19 |
+
use regex::Regex;
|
20 |
use serde::Deserialize;
|
21 |
use tokio::join;
|
22 |
|
23 |
+
// ---- Constants ----
|
24 |
+
/// Initialize redis cache connection once and store it on the heap.
|
25 |
+
static REDIS_CACHE: async_once_cell::OnceCell<RedisCache> = async_once_cell::OnceCell::new();
|
26 |
+
|
27 |
/// A named struct which deserializes all the user provided search parameters and stores them.
|
28 |
#[derive(Deserialize)]
|
29 |
struct SearchParams {
|
|
|
33 |
/// It stores the search parameter `page` (or pageno in simple words)
|
34 |
/// of the search url.
|
35 |
page: Option<u32>,
|
36 |
+
safesearch: Option<u8>,
|
37 |
}
|
38 |
|
39 |
/// Handles the route of index page or main page of the `websurfx` meta search engine website.
|
|
|
62 |
/// A named struct which is used to deserialize the cookies fetched from the client side.
|
63 |
#[allow(dead_code)]
|
64 |
#[derive(Deserialize)]
|
65 |
+
struct Cookie<'a> {
|
66 |
/// It stores the theme name used in the website.
|
67 |
+
theme: &'a str,
|
68 |
/// It stores the colorscheme name used for the website theme.
|
69 |
+
colorscheme: &'a str,
|
70 |
/// It stores the user selected upstream search engines selected from the UI.
|
71 |
+
engines: Vec<&'a str>,
|
72 |
}
|
73 |
|
74 |
/// Handles the route of search page of the `websurfx` meta search engine website and it takes
|
|
|
104 |
None => 1,
|
105 |
};
|
106 |
|
107 |
+
let safe_search: u8 = match config.safe_search {
|
108 |
+
3..=4 => config.safe_search,
|
109 |
+
_ => match ¶ms.safesearch {
|
110 |
+
Some(safesearch) => match safesearch {
|
111 |
+
0..=2 => *safesearch,
|
112 |
+
_ => 1,
|
113 |
+
},
|
114 |
+
None => config.safe_search,
|
115 |
+
},
|
116 |
+
};
|
117 |
+
|
118 |
let (_, results, _) = join!(
|
119 |
results(
|
120 |
format!(
|
121 |
+
"http://{}:{}/search?q={}&page={}&safesearch={}",
|
122 |
config.binding_ip,
|
123 |
config.port,
|
124 |
query,
|
125 |
+
page - 1,
|
126 |
+
safe_search
|
127 |
),
|
128 |
&config,
|
129 |
+
query,
|
130 |
page - 1,
|
131 |
req.clone(),
|
132 |
+
safe_search
|
133 |
),
|
134 |
results(
|
135 |
format!(
|
136 |
+
"http://{}:{}/search?q={}&page={}&safesearch={}",
|
137 |
+
config.binding_ip, config.port, query, page, safe_search
|
138 |
),
|
139 |
&config,
|
140 |
+
query,
|
141 |
page,
|
142 |
req.clone(),
|
143 |
+
safe_search
|
144 |
),
|
145 |
results(
|
146 |
format!(
|
147 |
+
"http://{}:{}/search?q={}&page={}&safesearch={}",
|
148 |
config.binding_ip,
|
149 |
config.port,
|
150 |
query,
|
151 |
+
page + 1,
|
152 |
+
safe_search
|
153 |
),
|
154 |
&config,
|
155 |
+
query,
|
156 |
page + 1,
|
157 |
req.clone(),
|
158 |
+
safe_search
|
159 |
)
|
160 |
);
|
161 |
|
|
|
186 |
async fn results(
|
187 |
url: String,
|
188 |
config: &Config,
|
189 |
+
query: &str,
|
190 |
page: u32,
|
191 |
req: HttpRequest,
|
192 |
+
safe_search: u8,
|
193 |
) -> Result<SearchResults, Box<dyn std::error::Error>> {
|
194 |
// Initialize redis cache connection struct
|
195 |
+
let mut redis_cache: RedisCache = REDIS_CACHE
|
196 |
+
.get_or_init(async {
|
197 |
+
// Initialize redis cache connection pool only one and store it in the heap.
|
198 |
+
RedisCache::new(&config.redis_url, 5).await.unwrap()
|
199 |
+
})
|
200 |
+
.await
|
201 |
+
.clone();
|
202 |
// fetch the cached results json.
|
203 |
+
let cached_results_json: Result<String, error_stack::Report<crate::cache::error::PoolError>> =
|
204 |
+
redis_cache.clone().cached_json(&url).await;
|
205 |
// check if fetched cache results was indeed fetched or it was an error and if so
|
206 |
// handle the data accordingly.
|
207 |
match cached_results_json {
|
208 |
+
Ok(results) => Ok(serde_json::from_str::<SearchResults>(&results)?),
|
209 |
Err(_) => {
|
210 |
+
if safe_search == 4 {
|
211 |
+
let mut results: SearchResults = SearchResults::default();
|
212 |
+
let mut _flag: bool =
|
213 |
+
is_match_from_filter_list(file_path(FileType::BlockList)?, query)?;
|
214 |
+
_flag = !is_match_from_filter_list(file_path(FileType::AllowList)?, query)?;
|
215 |
+
|
216 |
+
if _flag {
|
217 |
+
results.set_disallowed();
|
218 |
+
results.add_style(&config.style);
|
219 |
+
results.set_page_query(query);
|
220 |
+
redis_cache
|
221 |
+
.cache_results(&serde_json::to_string(&results)?, &url)
|
222 |
+
.await?;
|
223 |
+
return Ok(results);
|
224 |
+
}
|
225 |
+
}
|
226 |
+
|
227 |
// check if the cookie value is empty or not if it is empty then use the
|
228 |
// default selected upstream search engines from the config file otherwise
|
229 |
// parse the non-empty cookie and grab the user selected engines from the
|
230 |
// UI and use that.
|
231 |
+
let mut results: SearchResults = match req.cookie("appCookie") {
|
|
|
|
|
232 |
Some(cookie_value) => {
|
233 |
let cookie_value: Cookie = serde_json::from_str(cookie_value.name_value().1)?;
|
234 |
|
235 |
+
let engines: Vec<EngineHandler> = cookie_value
|
236 |
.engines
|
237 |
.iter()
|
238 |
.filter_map(|name| EngineHandler::new(name))
|
|
|
243 |
page,
|
244 |
config.aggregator.random_delay,
|
245 |
config.debug,
|
246 |
+
&engines,
|
247 |
config.request_timeout,
|
248 |
+
safe_search,
|
249 |
)
|
250 |
.await?
|
251 |
}
|
|
|
255 |
page,
|
256 |
config.aggregator.random_delay,
|
257 |
config.debug,
|
258 |
+
&config.upstream_search_engines,
|
259 |
config.request_timeout,
|
260 |
+
safe_search,
|
261 |
)
|
262 |
.await?
|
263 |
}
|
264 |
};
|
265 |
+
if results.engine_errors_info().is_empty() && results.results().is_empty() {
|
266 |
+
results.set_filtered();
|
267 |
+
}
|
268 |
+
results.add_style(&config.style);
|
269 |
+
redis_cache
|
270 |
+
.cache_results(&serde_json::to_string(&results)?, &url)
|
271 |
+
.await?;
|
272 |
Ok(results)
|
273 |
}
|
274 |
}
|
275 |
}
|
276 |
|
277 |
+
fn is_match_from_filter_list(
|
278 |
+
file_path: &str,
|
279 |
+
query: &str,
|
280 |
+
) -> Result<bool, Box<dyn std::error::Error>> {
|
281 |
+
let mut flag = false;
|
282 |
+
let mut reader = BufReader::new(File::open(file_path)?);
|
283 |
+
for line in reader.by_ref().lines() {
|
284 |
+
let re = Regex::new(&line?)?;
|
285 |
+
if re.is_match(query) {
|
286 |
+
flag = true;
|
287 |
+
break;
|
288 |
+
}
|
289 |
+
}
|
290 |
+
Ok(flag)
|
291 |
+
}
|
292 |
+
|
293 |
/// Handles the route of robots.txt page of the `websurfx` meta search engine website.
|
294 |
#[get("/robots.txt")]
|
295 |
pub async fn robots_data(_req: HttpRequest) -> Result<HttpResponse, Box<dyn std::error::Error>> {
|
websurfx/config.lua
CHANGED
@@ -10,6 +10,21 @@ production_use = false -- whether to use production mode or not (in other words
|
|
10 |
-- if production_use is set to true
|
11 |
-- There will be a random delay before sending the request to the search engines, this is to prevent DDoSing the upstream search engines from a large number of simultaneous requests.
|
12 |
request_timeout = 30 -- timeout for the search requests sent to the upstream search engines to be fetched (value in seconds).
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
13 |
|
14 |
-- ### Website ###
|
15 |
-- The different colorschemes provided are:
|
@@ -34,4 +49,7 @@ theme = "simple" -- the theme name which should be used for the website
|
|
34 |
redis_url = "redis://127.0.0.1:8082" -- redis connection url address on which the client should connect on.
|
35 |
|
36 |
-- ### Search Engines ###
|
37 |
-
upstream_search_engines = {
|
|
|
|
|
|
|
|
10 |
-- if production_use is set to true
|
11 |
-- There will be a random delay before sending the request to the search engines, this is to prevent DDoSing the upstream search engines from a large number of simultaneous requests.
|
12 |
request_timeout = 30 -- timeout for the search requests sent to the upstream search engines to be fetched (value in seconds).
|
13 |
+
rate_limiter = {
|
14 |
+
number_of_requests = 20, -- The number of request that are allowed within a provided time limit.
|
15 |
+
time_limit = 3, -- The time limit in which the quantity of requests that should be accepted.
|
16 |
+
}
|
17 |
+
|
18 |
+
-- ### Search ###
|
19 |
+
-- Filter results based on different levels. The levels provided are:
|
20 |
+
-- {{
|
21 |
+
-- 0 - None
|
22 |
+
-- 1 - Low
|
23 |
+
-- 2 - Moderate
|
24 |
+
-- 3 - High
|
25 |
+
-- 4 - Aggressive
|
26 |
+
-- }}
|
27 |
+
safe_search = 2
|
28 |
|
29 |
-- ### Website ###
|
30 |
-- The different colorschemes provided are:
|
|
|
49 |
redis_url = "redis://127.0.0.1:8082" -- redis connection url address on which the client should connect on.
|
50 |
|
51 |
-- ### Search Engines ###
|
52 |
+
upstream_search_engines = {
|
53 |
+
DuckDuckGo = true,
|
54 |
+
Searx = false,
|
55 |
+
} -- select the upstream search engines from which the results should be fetched.
|