neon_arch
commited on
Commit
•
12bfc52
1
Parent(s):
ac4e29a
🧹 chore: make clippy happy (#201)
Browse files- Cargo.lock +65 -25
- Cargo.toml +2 -2
- src/config/parser.rs +1 -1
- src/results/aggregation_models.rs +2 -2
- src/results/aggregator.rs +3 -3
- src/server/routes.rs +11 -7
Cargo.lock
CHANGED
@@ -288,6 +288,12 @@ version = "1.0.75"
|
|
288 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
289 |
checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6"
|
290 |
|
|
|
|
|
|
|
|
|
|
|
|
|
291 |
[[package]]
|
292 |
name = "askama_escape"
|
293 |
version = "0.10.3"
|
@@ -565,7 +571,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
565 |
checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4"
|
566 |
dependencies = [
|
567 |
"bytes 1.4.0",
|
|
|
568 |
"memchr",
|
|
|
|
|
|
|
569 |
]
|
570 |
|
571 |
[[package]]
|
@@ -1822,6 +1832,20 @@ dependencies = [
|
|
1822 |
"ws2_32-sys",
|
1823 |
]
|
1824 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1825 |
[[package]]
|
1826 |
name = "native-tls"
|
1827 |
version = "0.2.11"
|
@@ -2177,6 +2201,26 @@ dependencies = [
|
|
2177 |
"siphasher 0.3.11",
|
2178 |
]
|
2179 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2180 |
[[package]]
|
2181 |
name = "pin-project-lite"
|
2182 |
version = "0.2.13"
|
@@ -2454,12 +2498,21 @@ version = "0.23.3"
|
|
2454 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
2455 |
checksum = "4f49cdc0bb3f412bf8e7d1bd90fe1d9eb10bc5c399ba90973c14662a27b3f8ba"
|
2456 |
dependencies = [
|
|
|
|
|
|
|
2457 |
"combine",
|
|
|
|
|
2458 |
"itoa 1.0.9",
|
2459 |
"percent-encoding 2.3.0",
|
|
|
2460 |
"ryu",
|
2461 |
"sha1_smol",
|
2462 |
"socket2 0.4.9",
|
|
|
|
|
|
|
2463 |
"url 2.4.1",
|
2464 |
]
|
2465 |
|
@@ -2578,30 +2631,6 @@ dependencies = [
|
|
2578 |
"winreg 0.50.0",
|
2579 |
]
|
2580 |
|
2581 |
-
[[package]]
|
2582 |
-
name = "rlua"
|
2583 |
-
version = "0.19.7"
|
2584 |
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
2585 |
-
checksum = "5d33e5ba15c3d43178f283ed5863d4531e292fc0e56fb773f3bea45f18e3a42a"
|
2586 |
-
dependencies = [
|
2587 |
-
"bitflags 1.3.2",
|
2588 |
-
"bstr",
|
2589 |
-
"libc",
|
2590 |
-
"num-traits",
|
2591 |
-
"rlua-lua54-sys",
|
2592 |
-
]
|
2593 |
-
|
2594 |
-
[[package]]
|
2595 |
-
name = "rlua-lua54-sys"
|
2596 |
-
version = "0.1.6"
|
2597 |
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
2598 |
-
checksum = "7aafabafe1895cb4a2be81a56d7ff3d46bf4b5d2f9cfdbea2ed404cdabe96474"
|
2599 |
-
dependencies = [
|
2600 |
-
"cc",
|
2601 |
-
"libc",
|
2602 |
-
"pkg-config",
|
2603 |
-
]
|
2604 |
-
|
2605 |
[[package]]
|
2606 |
name = "rustc-demangle"
|
2607 |
version = "0.1.23"
|
@@ -3310,6 +3339,17 @@ dependencies = [
|
|
3310 |
"tokio-sync",
|
3311 |
]
|
3312 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
3313 |
[[package]]
|
3314 |
name = "tokio-sync"
|
3315 |
version = "0.1.8"
|
@@ -3674,12 +3714,12 @@ dependencies = [
|
|
3674 |
"log",
|
3675 |
"md5",
|
3676 |
"mimalloc",
|
|
|
3677 |
"once_cell",
|
3678 |
"rand 0.8.5",
|
3679 |
"redis",
|
3680 |
"regex",
|
3681 |
"reqwest 0.11.20",
|
3682 |
-
"rlua",
|
3683 |
"rusty-hook",
|
3684 |
"scraper",
|
3685 |
"serde",
|
|
|
288 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
289 |
checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6"
|
290 |
|
291 |
+
[[package]]
|
292 |
+
name = "arc-swap"
|
293 |
+
version = "1.6.0"
|
294 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
295 |
+
checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6"
|
296 |
+
|
297 |
[[package]]
|
298 |
name = "askama_escape"
|
299 |
version = "0.10.3"
|
|
|
571 |
checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4"
|
572 |
dependencies = [
|
573 |
"bytes 1.4.0",
|
574 |
+
"futures-core",
|
575 |
"memchr",
|
576 |
+
"pin-project-lite",
|
577 |
+
"tokio 1.32.0",
|
578 |
+
"tokio-util",
|
579 |
]
|
580 |
|
581 |
[[package]]
|
|
|
1832 |
"ws2_32-sys",
|
1833 |
]
|
1834 |
|
1835 |
+
[[package]]
|
1836 |
+
name = "mlua"
|
1837 |
+
version = "0.8.10"
|
1838 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1839 |
+
checksum = "0bb37b0ba91f017aa7ca2b98ef99496827770cd635b4a932a6047c5b4bbe678e"
|
1840 |
+
dependencies = [
|
1841 |
+
"bstr",
|
1842 |
+
"cc",
|
1843 |
+
"num-traits",
|
1844 |
+
"once_cell",
|
1845 |
+
"pkg-config",
|
1846 |
+
"rustc-hash",
|
1847 |
+
]
|
1848 |
+
|
1849 |
[[package]]
|
1850 |
name = "native-tls"
|
1851 |
version = "0.2.11"
|
|
|
2201 |
"siphasher 0.3.11",
|
2202 |
]
|
2203 |
|
2204 |
+
[[package]]
|
2205 |
+
name = "pin-project"
|
2206 |
+
version = "1.1.3"
|
2207 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
2208 |
+
checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422"
|
2209 |
+
dependencies = [
|
2210 |
+
"pin-project-internal",
|
2211 |
+
]
|
2212 |
+
|
2213 |
+
[[package]]
|
2214 |
+
name = "pin-project-internal"
|
2215 |
+
version = "1.1.3"
|
2216 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
2217 |
+
checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"
|
2218 |
+
dependencies = [
|
2219 |
+
"proc-macro2 1.0.66",
|
2220 |
+
"quote 1.0.33",
|
2221 |
+
"syn 2.0.29",
|
2222 |
+
]
|
2223 |
+
|
2224 |
[[package]]
|
2225 |
name = "pin-project-lite"
|
2226 |
version = "0.2.13"
|
|
|
2498 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
2499 |
checksum = "4f49cdc0bb3f412bf8e7d1bd90fe1d9eb10bc5c399ba90973c14662a27b3f8ba"
|
2500 |
dependencies = [
|
2501 |
+
"arc-swap",
|
2502 |
+
"async-trait",
|
2503 |
+
"bytes 1.4.0",
|
2504 |
"combine",
|
2505 |
+
"futures 0.3.28",
|
2506 |
+
"futures-util",
|
2507 |
"itoa 1.0.9",
|
2508 |
"percent-encoding 2.3.0",
|
2509 |
+
"pin-project-lite",
|
2510 |
"ryu",
|
2511 |
"sha1_smol",
|
2512 |
"socket2 0.4.9",
|
2513 |
+
"tokio 1.32.0",
|
2514 |
+
"tokio-retry",
|
2515 |
+
"tokio-util",
|
2516 |
"url 2.4.1",
|
2517 |
]
|
2518 |
|
|
|
2631 |
"winreg 0.50.0",
|
2632 |
]
|
2633 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2634 |
[[package]]
|
2635 |
name = "rustc-demangle"
|
2636 |
version = "0.1.23"
|
|
|
3339 |
"tokio-sync",
|
3340 |
]
|
3341 |
|
3342 |
+
[[package]]
|
3343 |
+
name = "tokio-retry"
|
3344 |
+
version = "0.3.0"
|
3345 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
3346 |
+
checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f"
|
3347 |
+
dependencies = [
|
3348 |
+
"pin-project",
|
3349 |
+
"rand 0.8.5",
|
3350 |
+
"tokio 1.32.0",
|
3351 |
+
]
|
3352 |
+
|
3353 |
[[package]]
|
3354 |
name = "tokio-sync"
|
3355 |
version = "0.1.8"
|
|
|
3714 |
"log",
|
3715 |
"md5",
|
3716 |
"mimalloc",
|
3717 |
+
"mlua",
|
3718 |
"once_cell",
|
3719 |
"rand 0.8.5",
|
3720 |
"redis",
|
3721 |
"regex",
|
3722 |
"reqwest 0.11.20",
|
|
|
3723 |
"rusty-hook",
|
3724 |
"scraper",
|
3725 |
"serde",
|
Cargo.toml
CHANGED
@@ -19,8 +19,8 @@ serde_json = {version="1.0.105"}
|
|
19 |
fake-useragent = {version="0.1.3"}
|
20 |
env_logger = {version="0.10.0"}
|
21 |
log = {version="0.4.20"}
|
22 |
-
|
23 |
-
redis = {version="0.23.3"}
|
24 |
md5 = {version="0.7.0"}
|
25 |
rand={version="0.8.5"}
|
26 |
once_cell = {version="1.18.0"}
|
|
|
19 |
fake-useragent = {version="0.1.3"}
|
20 |
env_logger = {version="0.10.0"}
|
21 |
log = {version="0.4.20"}
|
22 |
+
mlua = {version="0.8.10", features=["luajit"]}
|
23 |
+
redis = {version="0.23.3", features=["tokio-comp","connection-manager"]}
|
24 |
md5 = {version="0.7.0"}
|
25 |
rand={version="0.8.5"}
|
26 |
once_cell = {version="1.18.0"}
|
src/config/parser.rs
CHANGED
@@ -117,7 +117,7 @@ impl Config {
|
|
117 |
.get::<_, HashMap<String, bool>>("upstream_search_engines")?
|
118 |
.into_iter()
|
119 |
.filter_map(|(key, value)| value.then_some(key))
|
120 |
-
.filter_map(|engine| crate::engines::engine_models::EngineHandler::new(engine))
|
121 |
.collect(),
|
122 |
request_timeout: globals.get::<_, u8>("request_timeout")?,
|
123 |
threads,
|
|
|
117 |
.get::<_, HashMap<String, bool>>("upstream_search_engines")?
|
118 |
.into_iter()
|
119 |
.filter_map(|(key, value)| value.then_some(key))
|
120 |
+
.filter_map(|engine| crate::engines::engine_models::EngineHandler::new(&engine))
|
121 |
.collect(),
|
122 |
request_timeout: globals.get::<_, u8>("request_timeout")?,
|
123 |
threads,
|
src/results/aggregation_models.rs
CHANGED
@@ -132,9 +132,9 @@ impl SearchResults {
|
|
132 |
) -> Self {
|
133 |
Self {
|
134 |
results,
|
135 |
-
page_query,
|
136 |
style: Style::default(),
|
137 |
-
engine_errors_info,
|
138 |
disallowed: Default::default(),
|
139 |
filtered: Default::default(),
|
140 |
}
|
|
|
132 |
) -> Self {
|
133 |
Self {
|
134 |
results,
|
135 |
+
page_query: page_query.to_owned(),
|
136 |
style: Style::default(),
|
137 |
+
engine_errors_info: engine_errors_info.to_owned(),
|
138 |
disallowed: Default::default(),
|
139 |
filtered: Default::default(),
|
140 |
}
|
src/results/aggregator.rs
CHANGED
@@ -93,7 +93,7 @@ pub async fn aggregate(
|
|
93 |
tasks.push(tokio::spawn(async move {
|
94 |
search_engine
|
95 |
.results(
|
96 |
-
query,
|
97 |
page,
|
98 |
user_agent.clone(),
|
99 |
request_timeout,
|
@@ -162,13 +162,13 @@ pub async fn aggregate(
|
|
162 |
filter_with_lists(
|
163 |
&mut result_map,
|
164 |
&mut blacklist_map,
|
165 |
-
|
166 |
)?;
|
167 |
|
168 |
filter_with_lists(
|
169 |
&mut blacklist_map,
|
170 |
&mut result_map,
|
171 |
-
|
172 |
)?;
|
173 |
|
174 |
drop(blacklist_map);
|
|
|
93 |
tasks.push(tokio::spawn(async move {
|
94 |
search_engine
|
95 |
.results(
|
96 |
+
&query,
|
97 |
page,
|
98 |
user_agent.clone(),
|
99 |
request_timeout,
|
|
|
162 |
filter_with_lists(
|
163 |
&mut result_map,
|
164 |
&mut blacklist_map,
|
165 |
+
file_path(FileType::BlockList)?,
|
166 |
)?;
|
167 |
|
168 |
filter_with_lists(
|
169 |
&mut blacklist_map,
|
170 |
&mut result_map,
|
171 |
+
file_path(FileType::AllowList)?,
|
172 |
)?;
|
173 |
|
174 |
drop(blacklist_map);
|
src/server/routes.rs
CHANGED
@@ -22,7 +22,7 @@ use tokio::join;
|
|
22 |
|
23 |
// ---- Constants ----
|
24 |
/// Initialize redis cache connection once and store it on the heap.
|
25 |
-
|
26 |
|
27 |
/// A named struct which deserializes all the user provided search parameters and stores them.
|
28 |
///
|
@@ -184,7 +184,7 @@ async fn results(
|
|
184 |
req: HttpRequest,
|
185 |
safe_search: u8,
|
186 |
) -> Result<SearchResults, Box<dyn std::error::Error>> {
|
187 |
-
let redis_cache: RedisCache = REDIS_CACHE
|
188 |
.get_or_init(async {
|
189 |
// Initialize redis cache connection pool only one and store it in the heap.
|
190 |
RedisCache::new(&config.redis_url, 5).await.unwrap()
|
@@ -203,14 +203,16 @@ async fn results(
|
|
203 |
if safe_search == 4 {
|
204 |
let mut results: SearchResults = SearchResults::default();
|
205 |
let mut _flag: bool =
|
206 |
-
is_match_from_filter_list(
|
207 |
-
_flag = !is_match_from_filter_list(
|
208 |
|
209 |
if _flag {
|
210 |
results.set_disallowed();
|
211 |
results.add_style(&config.style);
|
212 |
-
results.set_page_query(
|
213 |
-
redis_cache
|
|
|
|
|
214 |
return Ok(results);
|
215 |
}
|
216 |
}
|
@@ -257,7 +259,9 @@ async fn results(
|
|
257 |
results.set_filtered();
|
258 |
}
|
259 |
results.add_style(&config.style);
|
260 |
-
redis_cache
|
|
|
|
|
261 |
Ok(results)
|
262 |
}
|
263 |
}
|
|
|
22 |
|
23 |
// ---- Constants ----
|
24 |
/// Initialize redis cache connection once and store it on the heap.
|
25 |
+
static REDIS_CACHE: async_once_cell::OnceCell<RedisCache> = async_once_cell::OnceCell::new();
|
26 |
|
27 |
/// A named struct which deserializes all the user provided search parameters and stores them.
|
28 |
///
|
|
|
184 |
req: HttpRequest,
|
185 |
safe_search: u8,
|
186 |
) -> Result<SearchResults, Box<dyn std::error::Error>> {
|
187 |
+
let mut redis_cache: RedisCache = REDIS_CACHE
|
188 |
.get_or_init(async {
|
189 |
// Initialize redis cache connection pool only one and store it in the heap.
|
190 |
RedisCache::new(&config.redis_url, 5).await.unwrap()
|
|
|
203 |
if safe_search == 4 {
|
204 |
let mut results: SearchResults = SearchResults::default();
|
205 |
let mut _flag: bool =
|
206 |
+
is_match_from_filter_list(file_path(FileType::BlockList)?, query)?;
|
207 |
+
_flag = !is_match_from_filter_list(file_path(FileType::AllowList)?, query)?;
|
208 |
|
209 |
if _flag {
|
210 |
results.set_disallowed();
|
211 |
results.add_style(&config.style);
|
212 |
+
results.set_page_query(query);
|
213 |
+
redis_cache
|
214 |
+
.cache_results(&serde_json::to_string(&results)?, &url)
|
215 |
+
.await?;
|
216 |
return Ok(results);
|
217 |
}
|
218 |
}
|
|
|
259 |
results.set_filtered();
|
260 |
}
|
261 |
results.add_style(&config.style);
|
262 |
+
redis_cache
|
263 |
+
.cache_results(&serde_json::to_string(&results)?, &url)
|
264 |
+
.await?;
|
265 |
Ok(results)
|
266 |
}
|
267 |
}
|