Repository: ALU-Schumacher/AUDITOR (language: rust)

Issue:
Slurm Collector: Handle job state / missing timestamps
## Fist part of the problem
Currently the collector naively takes a list of job states from the config file and tries to collect all corresponding jobs.
But obviously not all job states make sense in this context. E.g. `Pending` makes no sense.
I think that we should document a list of job states that can be sensibly used with this collector.

## Second part
There are job states that are a little more involved, like `Cancelled`. You might want to account for cancelled jobs when they were cancelled after running for a few days.
On the other hand there is no guarantee that a cancelled job was ever started. In this case the `start_time` is `Unknown` and tokenization of the sacct output will fail at
https://github.com/ALU-Schumacher/AUDITOR/blob/256395c0ebf67c208742ecf254c098ba0cd8ed7e/collectors/slurm/src/sacctcaller.rs#L206

I believe we need to define which fields might be missing for what job states and have the collector ignore certain entries. (Like a `Cancelled` job with no `start_time` instead of crashing.
I didn't think the list through (https://slurm.schedmd.com/sacct.html#SECTION_JOB-STATE-CODES). `Cancelled` might be the only problem.


Discussion / hints:

Fix patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index b58055f55..7b2242534 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -31,6 +31,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Dependencies: Update setuptools from 75.3.0 to 75.6.0 ([@dirksammel](https://github.com/dirksammel))
 - Dependencies: Update sqlx from 0.7.4 to 0.8.2 (missed some occurrences) ([@dirksammel](https://github.com/dirksammel))
 - Apel plugin: Update timestamp JSON atomically ([@maxfischer2781](https://github.com/maxfischer2781))
+- Slurm collector: Fix timezone offset of local timestamp `lastcheck` (#681, #178) ([@rkleinem](https://github.com/rkleinem))
+- Slurm collector: Ignore `.extern` steps instead of handling them as separate jobs (#812) ([@rkleinem](https://github.com/rkleinem))
+- Slurm collector: Ignore cancelled jobs which have never been started (#811) ([@rkleinem](https://github.com/rkleinem))
 
 ### Removed
 - Dependencies: Remove opentelemetry_api (replaced by opentelemetry) ([@dirksammel](https://github.com/dirksammel))
diff --git a/Cargo.lock b/Cargo.lock
index 52c4899e6..6ba83af7f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1033,6 +1033,17 @@ dependencies = [
  "subtle",
 ]
 
+[[package]]
+name = "displaydoc"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.85",
+]
+
 [[package]]
 name = "dlv-list"
 version = "0.3.0"
@@ -1399,12 +1410,6 @@ dependencies = [
  "hashbrown 0.14.5",
 ]
 
-[[package]]
-name = "heck"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
-
 [[package]]
 name = "heck"
 version = "0.5.0"
@@ -1594,6 +1599,124 @@ dependencies = [
  "cc",
 ]
 
+[[package]]
+name = "icu_collections"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526"
+dependencies = [
+ "displaydoc",
+ "yoke",
+ "zerofrom",
+ "zerovec",
+]
+
+[[package]]
+name = "icu_locid"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637"
+dependencies = [
+ "displaydoc",
+ "litemap",
+ "tinystr",
+ "writeable",
+ "zerovec",
+]
+
+[[package]]
+name = "icu_locid_transform"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e"
+dependencies = [
+ "displaydoc",
+ "icu_locid",
+ "icu_locid_transform_data",
+ "icu_provider",
+ "tinystr",
+ "zerovec",
+]
+
+[[package]]
+name = "icu_locid_transform_data"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e"
+
+[[package]]
+name = "icu_normalizer"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f"
+dependencies = [
+ "displaydoc",
+ "icu_collections",
+ "icu_normalizer_data",
+ "icu_properties",
+ "icu_provider",
+ "smallvec",
+ "utf16_iter",
+ "utf8_iter",
+ "write16",
+ "zerovec",
+]
+
+[[package]]
+name = "icu_normalizer_data"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516"
+
+[[package]]
+name = "icu_properties"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5"
+dependencies = [
+ "displaydoc",
+ "icu_collections",
+ "icu_locid_transform",
+ "icu_properties_data",
+ "icu_provider",
+ "tinystr",
+ "zerovec",
+]
+
+[[package]]
+name = "icu_properties_data"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569"
+
+[[package]]
+name = "icu_provider"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9"
+dependencies = [
+ "displaydoc",
+ "icu_locid",
+ "icu_provider_macros",
+ "stable_deref_trait",
+ "tinystr",
+ "writeable",
+ "yoke",
+ "zerofrom",
+ "zerovec",
+]
+
+[[package]]
+name = "icu_provider_macros"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.85",
+]
+
 [[package]]
 name = "ident_case"
 version = "1.0.1"
@@ -1602,12 +1725,23 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
 
 [[package]]
 name = "idna"
-version = "0.5.0"
+version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6"
+checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e"
 dependencies = [
- "unicode-bidi",
- "unicode-normalization",
+ "idna_adapter",
+ "smallvec",
+ "utf8_iter",
+]
+
+[[package]]
+name = "idna_adapter"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71"
+dependencies = [
+ "icu_normalizer",
+ "icu_properties",
 ]
 
 [[package]]
@@ -1770,6 +1904,12 @@ version = "0.4.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89"
 
+[[package]]
+name = "litemap"
+version = "0.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104"
+
 [[package]]
 name = "local-channel"
 version = "0.1.5"
@@ -2360,9 +2500,9 @@ checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94"
 
 [[package]]
 name = "pyo3"
-version = "0.20.3"
+version = "0.22.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "53bdbb96d49157e65d45cc287af5f32ffadd5f4761438b527b055fb0d4bb8233"
+checksum = "f402062616ab18202ae8319da13fa4279883a2b8a9d9f83f20dbade813ce1884"
 dependencies = [
  "anyhow",
  "cfg-if",
@@ -2370,7 +2510,7 @@ dependencies = [
  "indoc",
  "libc",
  "memoffset",
- "parking_lot",
+ "once_cell",
  "portable-atomic",
  "pyo3-build-config",
  "pyo3-ffi",
@@ -2379,35 +2519,35 @@ dependencies = [
 ]
 
 [[package]]
-name = "pyo3-asyncio"
-version = "0.20.0"
+name = "pyo3-async-runtimes"
+version = "0.22.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ea6b68e93db3622f3bb3bf363246cf948ed5375afe7abff98ccbdd50b184995"
+checksum = "2529f0be73ffd2be0cc43c013a640796558aa12d7ca0aab5cc14f375b4733031"
 dependencies = [
  "futures",
  "once_cell",
  "pin-project-lite",
  "pyo3",
- "pyo3-asyncio-macros",
+ "pyo3-async-runtimes-macros",
  "tokio",
 ]
 
 [[package]]
-name = "pyo3-asyncio-macros"
-version = "0.20.0"
+name = "pyo3-async-runtimes-macros"
+version = "0.22.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "56c467178e1da6252c95c29ecf898b133f742e9181dca5def15dc24e19d45a39"
+checksum = "22c26fd8e9fc19f53f0c1e00bf61471de6789f7eb263056f7f944a9cceb5823e"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 1.0.109",
+ "syn 2.0.85",
 ]
 
 [[package]]
 name = "pyo3-build-config"
-version = "0.20.3"
+version = "0.22.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "deaa5745de3f5231ce10517a1f5dd97d53e5a2fd77aa6b5842292085831d48d7"
+checksum = "b14b5775b5ff446dd1056212d778012cbe8a0fbffd368029fd9e25b514479c38"
 dependencies = [
  "once_cell",
  "target-lexicon",
@@ -2415,9 +2555,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-ffi"
-version = "0.20.3"
+version = "0.22.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62b42531d03e08d4ef1f6e85a2ed422eb678b8cd62b762e53891c05faf0d4afa"
+checksum = "9ab5bcf04a2cdcbb50c7d6105de943f543f9ed92af55818fd17b660390fc8636"
 dependencies = [
  "libc",
  "pyo3-build-config",
@@ -2425,9 +2565,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-macros"
-version = "0.20.3"
+version = "0.22.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7305c720fa01b8055ec95e484a6eca7a83c841267f0dd5280f0c8b8551d2c158"
+checksum = "0fd24d897903a9e6d80b968368a34e1525aeb719d568dba8b3d4bfa5dc67d453"
 dependencies = [
  "proc-macro2",
  "pyo3-macros-backend",
@@ -2437,11 +2577,11 @@ dependencies = [
 
 [[package]]
 name = "pyo3-macros-backend"
-version = "0.20.3"
+version = "0.22.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7c7e9b68bb9c3149c5b0cade5d07f953d6d125eb4337723c4ccdb665f1f96185"
+checksum = "36c011a03ba1e50152b4b394b479826cad97e7a21eb52df179cd91ac411cbfbe"
 dependencies = [
- "heck 0.4.1",
+ "heck",
  "proc-macro2",
  "pyo3-build-config",
  "quote",
@@ -2457,7 +2597,7 @@ dependencies = [
  "auditor-client",
  "chrono",
  "pyo3",
- "pyo3-asyncio",
+ "pyo3-async-runtimes",
  "serde",
  "serde_json",
  "tokio",
@@ -3167,7 +3307,7 @@ checksum = "1804e8a7c7865599c9c79be146dc8a9fd8cc86935fa641d3ea58e5f0688abaa5"
 dependencies = [
  "dotenvy",
  "either",
- "heck 0.5.0",
+ "heck",
  "hex",
  "once_cell",
  "proc-macro2",
@@ -3294,6 +3434,12 @@ dependencies = [
  "uuid",
 ]
 
+[[package]]
+name = "stable_deref_trait"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
+
 [[package]]
 name = "stringprep"
 version = "0.1.5"
@@ -3348,6 +3494,17 @@ dependencies = [
  "futures-core",
 ]
 
+[[package]]
+name = "synstructure"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.85",
+]
+
 [[package]]
 name = "target-lexicon"
 version = "0.12.16"
@@ -3428,6 +3585,16 @@ dependencies = [
  "time-core",
 ]
 
+[[package]]
+name = "tinystr"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f"
+dependencies = [
+ "displaydoc",
+ "zerovec",
+]
+
 [[package]]
 name = "tinytemplate"
 version = "1.2.1"
@@ -3717,9 +3884,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
 
 [[package]]
 name = "url"
-version = "2.5.2"
+version = "2.5.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c"
+checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60"
 dependencies = [
  "form_urlencoded",
  "idna",
@@ -3732,6 +3899,18 @@ version = "2.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da"
 
+[[package]]
+name = "utf16_iter"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246"
+
+[[package]]
+name = "utf8_iter"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
+
 [[package]]
 name = "uuid"
 version = "1.11.0"
@@ -4128,6 +4307,18 @@ dependencies = [
  "url",
 ]
 
+[[package]]
+name = "write16"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936"
+
+[[package]]
+name = "writeable"
+version = "0.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51"
+
 [[package]]
 name = "yaml-rust"
 version = "0.4.5"
@@ -4137,6 +4328,30 @@ dependencies = [
  "linked-hash-map",
 ]
 
+[[package]]
+name = "yoke"
+version = "0.7.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40"
+dependencies = [
+ "serde",
+ "stable_deref_trait",
+ "yoke-derive",
+ "zerofrom",
+]
+
+[[package]]
+name = "yoke-derive"
+version = "0.7.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.85",
+ "synstructure",
+]
+
 [[package]]
 name = "zerocopy"
 version = "0.7.35"
@@ -4158,12 +4373,55 @@ dependencies = [
  "syn 2.0.85",
 ]
 
+[[package]]
+name = "zerofrom"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cff3ee08c995dee1859d998dea82f7374f2826091dd9cd47def953cae446cd2e"
+dependencies = [
+ "zerofrom-derive",
+]
+
+[[package]]
+name = "zerofrom-derive"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.85",
+ "synstructure",
+]
+
 [[package]]
 name = "zeroize"
 version = "1.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde"
 
+[[package]]
+name = "zerovec"
+version = "0.10.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079"
+dependencies = [
+ "yoke",
+ "zerofrom",
+ "zerovec-derive",
+]
+
+[[package]]
+name = "zerovec-derive"
+version = "0.10.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.85",
+]
+
 [[package]]
 name = "zstd"
 version = "0.13.2"
diff --git a/collectors/slurm/src/sacctcaller.rs b/collectors/slurm/src/sacctcaller.rs
index ff11d66dc..b37009def 100644
--- a/collectors/slurm/src/sacctcaller.rs
+++ b/collectors/slurm/src/sacctcaller.rs
@@ -12,7 +12,7 @@ use auditor::{
     constants::FORBIDDEN_CHARACTERS,
     domain::{Component, RecordAdd, Score},
 };
-use chrono::{DateTime, FixedOffset, Local, Utc};
+use chrono::{DateTime, Local, Utc};
 use color_eyre::eyre::{eyre, Result};
 use itertools::Itertools;
 use once_cell::sync::Lazy;
@@ -23,7 +23,7 @@ use crate::{
     configuration::{AllowedTypes, ComponentConfig, KeyConfig, ParsableType, Settings},
     database::Database,
     shutdown::Shutdown,
-    CONFIG, END, GROUP, JOBID, KEYS, START, USER,
+    CONFIG, END, GROUP, JOBID, KEYS, START, STATE, USER,
 };
 
 type SacctRow = HashMap<String, Option<AllowedTypes>>;
@@ -36,7 +36,12 @@ static BATCH_REGEX: Lazy<Regex> = Lazy::new(|| {
 });
 
 static SUB_REGEX: Lazy<Regex> = Lazy::new(|| {
-    Regex::new(r"^[0-9]+\.[0-9]*$")
+    Regex::new(r"^[0-9_]+\.[0-9]*$")
+        .expect("Could not construct essential Regex for matching job ids.")
+});
+
+static EXTERN_REGEX: Lazy<Regex> = Lazy::new(|| {
+    Regex::new(r"^[0-9_]+\.extern$")
         .expect("Could not construct essential Regex for matching job ids.")
 });
 
@@ -164,7 +169,6 @@ async fn get_job_info(database: &Database) -> Result<Vec<RecordAdd>> {
     let (nextcheck, rid) = if records.is_empty() {
         (lastcheck, last_record_id)
     } else {
-        let local_offset = Local::now().offset().utc_minus_local();
         let (ts, rid) = records.iter().fold(
             (chrono::DateTime::<Utc>::MIN_UTC, String::new()),
             |(acc, _acc_record_id), r| {
@@ -175,10 +179,7 @@ async fn get_job_info(database: &Database) -> Result<Vec<RecordAdd>> {
             },
         );
         (
-            DateTime::<Local>::from_naive_utc_and_offset(
-                ts.naive_utc(),
-                FixedOffset::east_opt(local_offset).unwrap(),
-            ),
+            DateTime::<Local>::from_naive_utc_and_offset(ts.naive_utc(), *Local::now().offset()),
             rid,
         )
     };
@@ -227,37 +228,62 @@ fn tokenize_sacct_output(output: &str, keys: Vec<KeyConfig>) -> SacctRows {
 #[tracing::instrument(name = "Parse sacct rows", skip(sacct_rows, keys))]
 fn parse_sacct_rows(sacct_rows: SacctRows, keys: &[KeyConfig]) -> Result<Vec<Job>> {
     tracing::debug!("sacct_rows = {:?}", sacct_rows);
-    sacct_rows
+    let mut jobs = Vec::with_capacity(sacct_rows.len());
+    for id in sacct_rows
         .keys()
         .filter(|k| !BATCH_REGEX.is_match(k))
         .filter(|k| !SUB_REGEX.is_match(k))
-        .map(|id| -> Result<Job> {
-            let map1 = sacct_rows.get(id).ok_or(eyre!("Cannot get map1"))?;
-            let map2 = sacct_rows.get(&format!("{id}.batch"));
-            Ok(keys.iter()
-                .cloned()
-                .filter_map(|KeyConfig {name: k, key_type: _, allow_empty: _}| {
-                    let val = match map1.get(&k) {
-                        Some(Some(v)) => Some(v.clone()),
-                        _ => {
-                            if let Some(map2) = map2 {
-                                match map2.get(&k) {
-                                    Some(Some(v)) => Some(v.clone()),
-                                    _ => {
-                                        tracing::error!("Something went wrong during parsing (map1, id: {id}, key: {k}, value: {:?})", map2.get(&k));
-                                        None
-                                    },
-                                }
-                            } else {
-                                tracing::error!("Something went wrong during parsing (map2, id: {id}, key: {k}, value: {:?})", map1.get(&k));
-                                None
+        .filter(|k| !EXTERN_REGEX.is_match(k))
+    {
+        let map1 = sacct_rows.get(id).ok_or(eyre!("Cannot get map1"))?;
+        let map2 = sacct_rows.get(&format!("{id}.batch"));
+        // A state might look like "CANCELLED by 1000"
+        let cancelled = map1
+            .get(STATE)
+            .and_then(Option::as_ref)
+            .ok_or(eyre!("Job {} has no state.", id))?
+            .extract_string()?
+            .starts_with("CANCELLED");
+        if cancelled
+            && map1.get(START).and_then(Option::as_ref).is_none()
+            && map2
+                .and_then(|m| m.get(START))
+                .and_then(Option::as_ref)
+                .is_none()
+        {
+            tracing::debug!(
+                "Ignore Job {} since it was cancelled before it was started.",
+                id
+            );
+            continue;
+        };
+        let job = keys
+            .iter()
+            .cloned()
+            .filter_map(|KeyConfig {name: k, key_type: _, allow_empty: _}| {
+                let val = match map1.get(&k) {
+                    Some(Some(v)) => Some(v.clone()),
+                    _ => {
+                        if let Some(map2) = map2 {
+                            match map2.get(&k) {
+                                Some(Some(v)) => Some(v.clone()),
+                                _ => {
+                                    tracing::error!("Something went wrong during parsing (map1, id: {id}, key: {k}, value: {:?})", map2.get(&k));
+                                    None
+                                },
                             }
-                        },
-                    };
-                    val.map(|val| (k, val))
-                })
-                .collect::<Job>())
-        }).collect::<Result<Vec<Job>>>()
+                        } else {
+                            tracing::error!("Something went wrong during parsing (map2, id: {id}, key: {k}, value: {:?})", map1.get(&k));
+                            None
+                        }
+                    },
+                };
+                val.map(|val| (k, val))
+            })
+            .collect::<Job>();
+        jobs.push(job);
+    }
+    Ok(jobs)
 }
 
 #[tracing::instrument(
@@ -450,7 +476,7 @@ fn construct_component_scores(job: &Job, component_config: &ComponentConfig) ->
 #[cfg(test)]
 mod tests {
     use auditor::domain::{ValidAmount, ValidName, ValidValue};
-    use chrono::NaiveDateTime;
+    use chrono::{FixedOffset, NaiveDateTime};
 
     use super::*;
     use crate::{
@@ -983,6 +1009,221 @@ mod tests {
         assert_eq!(parsed_sacct_rows, expected);
     }
 
+    #[test]
+    fn parse_sacct_rows_unstarted_cancelled_skipped() {
+        let keys = vec![
+            KeyConfig {
+                name: JOBID.to_owned(),
+                key_type: ParsableType::String,
+                allow_empty: false,
+            },
+            KeyConfig {
+                name: START.to_owned(),
+                key_type: ParsableType::DateTime,
+                allow_empty: false,
+            },
+            KeyConfig {
+                name: END.to_owned(),
+                key_type: ParsableType::DateTime,
+                allow_empty: false,
+            },
+            KeyConfig {
+                name: STATE.to_owned(),
+                key_type: ParsableType::String,
+                allow_empty: false,
+            },
+        ];
+
+        let sacct_rows = SacctRows::from([
+            (
+                "1234567".to_owned(),
+                SacctRow::from([
+                    (
+                        JOBID.to_owned(),
+                        Some(AllowedTypes::String("1234567".to_owned())),
+                    ),
+                    (START.to_owned(), None),
+                    (END.to_owned(), None),
+                    (
+                        STATE.to_owned(),
+                        Some(AllowedTypes::String("CANCELLED by 1000".to_owned())),
+                    ),
+                ]),
+            ),
+            (
+                "1234567.batch".to_owned(),
+                SacctRow::from([
+                    (
+                        JOBID.to_owned(),
+                        Some(AllowedTypes::String("1234567.batch".to_owned())),
+                    ),
+                    (START.to_owned(), None),
+                    (END.to_owned(), None),
+                    (
+                        STATE.to_owned(),
+                        Some(AllowedTypes::String("CANCELLED by 1000".to_owned())),
+                    ),
+                ]),
+            ),
+        ]);
+
+        let parsed_sacct_rows = parse_sacct_rows(sacct_rows, &keys).unwrap();
+
+        let expected = vec![];
+        assert_eq!(parsed_sacct_rows, expected);
+    }
+
+    #[test]
+    fn parse_sacct_rows_started_cancelled_succeeds() {
+        let keys = vec![
+            KeyConfig {
+                name: JOBID.to_owned(),
+                key_type: ParsableType::String,
+                allow_empty: false,
+            },
+            KeyConfig {
+                name: START.to_owned(),
+                key_type: ParsableType::DateTime,
+                allow_empty: false,
+            },
+            KeyConfig {
+                name: END.to_owned(),
+                key_type: ParsableType::DateTime,
+                allow_empty: false,
+            },
+            KeyConfig {
+                name: STATE.to_owned(),
+                key_type: ParsableType::String,
+                allow_empty: false,
+            },
+        ];
+
+        let sacct_rows = SacctRows::from([
+            (
+                "1234567".to_owned(),
+                SacctRow::from([
+                    (
+                        JOBID.to_owned(),
+                        Some(AllowedTypes::String("1234567".to_owned())),
+                    ),
+                    (
+                        START.to_owned(),
+                        Some(AllowedTypes::DateTime(DateTime::<Utc>::from(
+                            NaiveDateTime::parse_from_str(
+                                "2023-11-07T10:14:01",
+                                "%Y-%m-%dT%H:%M:%S",
+                            )
+                            .unwrap()
+                            .and_local_timezone(
+                                FixedOffset::east_opt(Local::now().offset().local_minus_utc())
+                                    .unwrap(),
+                            )
+                            .unwrap(),
+                        ))),
+                    ),
+                    (
+                        END.to_owned(),
+                        Some(AllowedTypes::DateTime(DateTime::<Utc>::from(
+                            NaiveDateTime::parse_from_str(
+                                "2023-11-07T11:39:09",
+                                "%Y-%m-%dT%H:%M:%S",
+                            )
+                            .unwrap()
+                            .and_local_timezone(
+                                FixedOffset::east_opt(Local::now().offset().local_minus_utc())
+                                    .unwrap(),
+                            )
+                            .unwrap(),
+                        ))),
+                    ),
+                    (
+                        STATE.to_owned(),
+                        Some(AllowedTypes::String("CANCELLED by 1000".to_owned())),
+                    ),
+                ]),
+            ),
+            (
+                "1234567.batch".to_owned(),
+                SacctRow::from([
+                    (
+                        JOBID.to_owned(),
+                        Some(AllowedTypes::String("1234567.batch".to_owned())),
+                    ),
+                    (
+                        START.to_owned(),
+                        Some(AllowedTypes::DateTime(DateTime::<Utc>::from(
+                            NaiveDateTime::parse_from_str(
+                                "2023-11-07T10:14:01",
+                                "%Y-%m-%dT%H:%M:%S",
+                            )
+                            .unwrap()
+                            .and_local_timezone(
+                                FixedOffset::east_opt(Local::now().offset().local_minus_utc())
+                                    .unwrap(),
+                            )
+                            .unwrap(),
+                        ))),
+                    ),
+                    (
+                        END.to_owned(),
+                        Some(AllowedTypes::DateTime(DateTime::<Utc>::from(
+                            NaiveDateTime::parse_from_str(
+                                "2023-11-07T11:39:09",
+                                "%Y-%m-%dT%H:%M:%S",
+                            )
+                            .unwrap()
+                            .and_local_timezone(
+                                FixedOffset::east_opt(Local::now().offset().local_minus_utc())
+                                    .unwrap(),
+                            )
+                            .unwrap(),
+                        ))),
+                    ),
+                    (
+                        STATE.to_owned(),
+                        Some(AllowedTypes::String("CANCELLED by 1000".to_owned())),
+                    ),
+                ]),
+            ),
+        ]);
+
+        let parsed_sacct_rows = parse_sacct_rows(sacct_rows, &keys).unwrap();
+
+        let expected = vec![Job::from([
+            (
+                "JobID".to_owned(),
+                AllowedTypes::String("1234567".to_owned()),
+            ),
+            (
+                START.to_owned(),
+                AllowedTypes::DateTime(DateTime::<Utc>::from(
+                    NaiveDateTime::parse_from_str("2023-11-07T10:14:01", "%Y-%m-%dT%H:%M:%S")
+                        .unwrap()
+                        .and_local_timezone(
+                            FixedOffset::east_opt(Local::now().offset().local_minus_utc()).unwrap(),
+                        )
+                        .unwrap(),
+                )),
+            ),
+            (
+                END.to_owned(),
+                AllowedTypes::DateTime(DateTime::<Utc>::from(
+                    NaiveDateTime::parse_from_str("2023-11-07T11:39:09", "%Y-%m-%dT%H:%M:%S")
+                        .unwrap()
+                        .and_local_timezone(
+                            FixedOffset::east_opt(Local::now().offset().local_minus_utc()).unwrap(),
+                        )
+                        .unwrap(),
+                )),
+            ),
+            (
+                STATE.to_owned(),
+                AllowedTypes::String("CANCELLED by 1000".to_owned()),
+            ),
+        ])];
+        assert_eq!(parsed_sacct_rows, expected);
+    }
+
     #[test]
     fn construct_components_empty_config_succeeds() {
         let job = Job::from([(
diff --git a/deny.toml b/deny.toml
index eaf32f216..608c41d7a 100644
--- a/deny.toml
+++ b/deny.toml
@@ -81,6 +81,7 @@ allow = [
     "ISC",
     "OpenSSL",
     "CC0-1.0",
+    "Unicode-3.0"
 ]
 # The confidence threshold for detecting a license from license text.
 # The higher the value, the more closely the license text must be to the


Test patch: