Domain: Biology — Ecology

Problem:
Given a synthetic multi-omics dataset of 200 human subjects (gut_brain1.csv) with the variables provided, construct a microbe–metabolite–phenotype causal network using significant associations of microbial taxa and metabolites with each neurocognitive phenotype using given model.  
Perform cross-validation and bootstrapping to assess the stability of associations. Use a random seed of \(42\). Perform \(5\)-fold cross-validation and \(1000\) bootstrap iterations. Handle missing values via median imputation. Standardize metabolites and phenotypes to z-scores.

Model details:

Sparse Multi-Omics Regression

\[Y_i = \alpha + X_i^{\text{CLR}} \beta_X + Z_i \beta_Z + C_i \gamma + \epsilon_i\]

Where:

- \(X_i^{\text{CLR}}\) = CLR-transformed microbiome vector for subject \(i\)  
- \(Z_i\) = metabolite vector for subject \(i\)  
- \(\beta_X, \beta_Z\) = effect sizes of taxa and metabolites (to be estimated sparsely)  
- \(\gamma\) = effects of host covariates  
- \(\epsilon_i \sim \mathcal{N}(0, \sigma^2)\) 
- Host covariates \(C_i = [\text{Age, Sex, BMI, Diet, Med}]\)

Sparsity via LASSO:

\[\min_\beta \|Y - X \beta\|_2^2 + \lambda \|\beta\|_1\]

Causal Network:

Input (Predictor): Significant microbial taxa (\(X_j\)); 
Mediator: Significant metabolites (\(Z_k\));  
Output (Response): Neurocognitive phenotypes (\(Y_i\)) - Anxiety, Depression, or Cognition

Give the final result highlighting the overall strongest potential therapeutic targets, both microbes and metabolites that appear central in mediating effects among anxiety, depression and cognition based on the TIS score as follows. Give result with \(3\) decimal precision in array format as [Node: TIS, Phenotype]

\[
\text{Therapeutic Importance Score} = |\text{Total Effect on Phenotype}| \times \text{Centrality Measure}
\]

Where:

- \(\text{Total Effect on Phenotype}\) = absolute value of the total effect (direct + indirect)  
- \(\text{Centrality Measure}\) = network centrality metric (e.g., degree, betweenness, or eigenvector centrality) in the taxa-metabolite-phenotype network

Answer: \(Taxa2:0.146-CogS\), \(Metab9: 0.001- DepS\)

Solution code:
# ====================================================
# GUT–BRAIN MULTI-OMICS CAUSAL NETWORK ANALYSIS (numpy/pandas/scipy only)
# ====================================================
import numpy as np
import pandas as pd
from scipy.stats import pearsonr
from scipy import linalg
import random
import math
import time

# -------------------------------
# GLOBAL CONFIGURATION
# -------------------------------
SEED = 42
np.random.seed(SEED)
random.seed(SEED)

ALPHA_GRID = np.logspace(-4, 0, 30)  # grid for alpha search
K = 5                                # CV folds
B = 1000                             # bootstrap iterations
corr_threshold = 0.3                 # edge inclusion threshold
MAX_ITER = 2000
TOL = 1e-6

# -------------------------------
# HELPERS: Coordinate Descent LASSO
# -------------------------------
def standardize_X(X):
    """Center columns and scale to unit L2 norm (if constant column -> keep zeros)."""
    X = X.astype(float)
    col_means = X.mean(axis=0)
    Xc = X - col_means
    col_norms = np.sqrt((Xc ** 2).sum(axis=0))
    # avoid division by zero
    col_norms_safe = np.where(col_norms == 0, 1.0, col_norms)
    Xs = Xc / col_norms_safe
    return Xs, col_means, col_norms_safe

def center_y(y):
    mu = y.mean()
    return y - mu, mu

def soft_threshold(rho, lam):
    if rho > 0 and lam < abs(rho):
        return rho - lam
    if rho < 0 and lam < abs(rho):
        return rho + lam
    return 0.0

def lasso_coordinate_descent(X, y, alpha, max_iter=MAX_ITER, tol=TOL):
    """
    LASSO via coordinate descent on standardized X (columns unit norm) and centered y.
    alpha is the L1 penalty (we interpret it as lambda / n in usual notation).
    Returns coefficients on standardized X scale.
    """
    n, p = X.shape
    beta = np.zeros(p, dtype=float)
    Xy = X.T.dot(y)  # helpful
    # Precompute column squared norms (should be 1 if standardized)
    col_norm2 = (X ** 2).sum(axis=0)
    for it in range(max_iter):
        beta_old = beta.copy()
        for j in range(p):
            # residual excluding j
            r_j = y - X.dot(beta) + X[:, j] * beta[j]
            rho = X[:, j].dot(r_j)
            # coordinate update (since X columns standardized, denom = col_norm2[j])
            z = col_norm2[j]
            # Soft thresholding
            bj = soft_threshold(rho / z, alpha / z)
            beta[j] = bj
        # check convergence
        if np.max(np.abs(beta - beta_old)) < tol:
            break
    return beta

def lasso_fit_with_alpha(X, y, alpha):
    """
    Standardize X and center y, fit LASSO coordinate descent, and return coefficients
    mapped back to original column scale (so coefficients correspond to original X).
    """
    Xs, x_means, x_norms = standardize_X(X)
    yc, y_mean = center_y(y)
    beta_s = lasso_coordinate_descent(Xs, yc, alpha)
    # map back to original scale: beta_orig = beta_s / col_norms
    beta_orig = beta_s / x_norms
    intercept = y_mean - np.dot(x_means, beta_orig)
    return beta_orig, intercept

# -------------------------------
# CROSS-VALIDATION to choose alpha
# -------------------------------
def lasso_cv_select_alpha(X, y, alphas=ALPHA_GRID, n_folds=K, seed=SEED):
    n = X.shape[0]
    indices = np.arange(n)
    np.random.RandomState(seed).shuffle(indices)
    folds = np.array_split(indices, n_folds)
    alpha_mse = []
    for alpha in alphas:
        mses = []
        for k in range(n_folds):
            val_idx = folds[k]
            train_idx = np.hstack([folds[i] for i in range(n_folds) if i != k])
            Xtr, ytr = X[train_idx], y[train_idx]
            Xv, yv = X[val_idx], y[val_idx]
            try:
                beta, intercept = lasso_fit_with_alpha(Xtr, ytr, alpha)
                preds = Xv.dot(beta) + intercept
                mse = np.mean((yv - preds) ** 2)
            except Exception:
                mse = np.inf
            mses.append(mse)
        alpha_mse.append(np.mean(mses))
    best_idx = int(np.argmin(alpha_mse))
    return alphas[best_idx]

# -------------------------------
# Utilities: correlation edges & centrality (degree)
# -------------------------------
def corr_edges_named(X_df, Z_df, threshold, X_names, Z_names):
    """Return list of edges (taxa_name, metab_name, corr) where abs(corr) > threshold."""
    edges = []
    if X_df.shape[1] == 0 or Z_df.shape[1] == 0:
        return edges
    for i, xi in enumerate(X_df.T):
        for j, zj in enumerate(Z_df.T):
            # handle constant arrays (pearsonr raises)
            if np.allclose(xi, xi[0]) or np.allclose(zj, zj[0]):
                continue
            r, _ = pearsonr(xi, zj)
            if abs(r) > threshold:
                edges.append((X_names[i], Z_names[j], r))
    return edges

def degree_centrality_from_edges(nodes_list, edges):
    """
    nodes_list: list of node names
    edges: list of (u,v,weight) for undirected or directed edges; treat as undirected for degree centrality
    degree centrality = degree / (n-1)
    We count unique neighbors (ignoring multiple parallel edges).
    """
    n = len(nodes_list)
    name_to_idx = {n_: i for i, n_ in enumerate(nodes_list)}
    neighbors = {name: set() for name in nodes_list}
    for u, v, w in edges:
        if u in neighbors and v in neighbors:
            neighbors[u].add(v)
            neighbors[v].add(u)
    deg_cent = {}
    denom = (n - 1) if n > 1 else 1.0
    for name in nodes_list:
        deg = len(neighbors[name])
        deg_cent[name] = deg / denom
    return deg_cent

# -------------------------------
# LOAD DATA
# -------------------------------
# Use developer-provided path
df = pd.read_csv("gut_brain1.csv")

# Identify column groups
microbe_cols = [c for c in df.columns if c.startswith("Taxa_")]
metab_cols = [c for c in df.columns if c.startswith("Metab_")]
phen_cols = ["AnxScore", "DepScore", "CogScore"]
cov_cols = ["Age", "Sex", "BMI", "Diet", "Med"]

# Handle missing numeric values with median (replicates previous behavior)
df = df.copy()
num_cols = df.select_dtypes(include=[np.number]).columns.tolist()
# apply median imputation for all numeric columns used below
medians = df[num_cols].median()
df[num_cols] = df[num_cols].fillna(medians)

# Standardize metabolites + phenotypes (z-score)
for col in metab_cols + phen_cols:
    if col in df.columns:
        col_vals = df[col].astype(float).values
        mu = col_vals.mean()
        sd = col_vals.std(ddof=0) if col_vals.std(ddof=0) != 0 else 1.0
        df[col] = (col_vals - mu) / sd

# -------------------------------
# MAIN LOOP OVER PHENOTYPES
# -------------------------------
all_targets = []

start_time = time.time()
for phen in phen_cols:
    print(f"\nProcessing phenotype: {phen}")
    X = df[microbe_cols].values if len(microbe_cols) > 0 else np.zeros((len(df), 0))
    Z = df[metab_cols].values if len(metab_cols) > 0 else np.zeros((len(df), 0))
    Y = df[phen].values
    # covariates retained but not penalized in original pipeline; we keep them unused in LASSO as before
    # (the earlier script passed C but didn't use them inside LassoCV, so we mimic that behavior)

    # LASSO-CV to choose alpha on original X (microbes) and Z (metabs) separately
    # If there are no columns, skip
    beta_X = np.zeros(X.shape[1]) if X.shape[1] > 0 else np.zeros(0)
    beta_Z = np.zeros(Z.shape[1]) if Z.shape[1] > 0 else np.zeros(0)

    if X.shape[1] > 0:
        best_alpha_X = lasso_cv_select_alpha(X, Y)
        # Bootstrap with chosen alpha
        coefs_X_boot = np.zeros((B, X.shape[1]))
        for b in range(B):
            idx = np.random.choice(X.shape[0], X.shape[0], replace=True)
            Xb, yb = X[idx, :], Y[idx]
            beta_b, intercept_b = lasso_fit_with_alpha(Xb, yb, best_alpha_X)
            coefs_X_boot[b, :] = beta_b
        beta_X = np.mean(coefs_X_boot, axis=0)

    if Z.shape[1] > 0:
        best_alpha_Z = lasso_cv_select_alpha(Z, Y)
        coefs_Z_boot = np.zeros((B, Z.shape[1]))
        for b in range(B):
            idx = np.random.choice(Z.shape[0], Z.shape[0], replace=True)
            Zb, yb = Z[idx, :], Y[idx]
            beta_b, intercept_b = lasso_fit_with_alpha(Zb, yb, best_alpha_Z)
            coefs_Z_boot[b, :] = beta_b
        beta_Z = np.mean(coefs_Z_boot, axis=0)

    # Select significant (non-zero) features using same threshold as original code
    sig_X = [microbe_cols[i] for i, b in enumerate(beta_X) if abs(b) > 0.01] if X.shape[1] > 0 else []
    sig_Z = [metab_cols[i] for i, b in enumerate(beta_Z) if abs(b) > 0.01] if Z.shape[1] > 0 else []

    if len(sig_X) == 0 and len(sig_Z) == 0:
        print(" No significant features for this phenotype.")
        continue

    X_sel_df = df[sig_X] if len(sig_X) > 0 else pd.DataFrame(np.zeros((len(df), 0)))
    Z_sel_df = df[sig_Z] if len(sig_Z) > 0 else pd.DataFrame(np.zeros((len(df), 0)))

    # Build edge list (use actual names)
    edges = []

    # Microbe–metabolite correlations (named)
    corr_list = corr_edges_named(X_sel_df.values, Z_sel_df.values, corr_threshold, sig_X, sig_Z)
    for (u, v, r) in corr_list:
        edges.append((u, v, r))

    # Microbe/metab -> phenotype edges using bootstrap mean betas
    for i, b in enumerate(beta_X):
        if abs(b) > 0.01:
            edges.append((microbe_cols[i], phen, float(b)))
    for j, b in enumerate(beta_Z):
        if abs(b) > 0.01:
            edges.append((metab_cols[j], phen, float(b)))

    # Build node list for centrality: include sig_X, sig_Z, phenotype
    nodes = list(dict.fromkeys(sig_X + sig_Z + [phen]))  # preserve order, unique

    # Compute degree centrality manually (undirected neighbors)
    deg_cent = degree_centrality_from_edges(nodes, edges)

    # Compute therapeutic scores per node as in original (sum abs weights of incident edges * degree centrality)
    for node in nodes:
        if node == phen:
            continue
        # determine node type
        ntype = "microbe" if node in microbe_cols else ("metabolite" if node in metab_cols else "other")
        # compute total effect = sum abs(weight) for edges incident to node
        total_effect = 0.0
        for (u, v, w) in edges:
            if u == node or v == node:
                total_effect += abs(w)
        centrality_val = deg_cent.get(node, 0.0)
        therapeutic_score = abs(total_effect) * centrality_val
        all_targets.append({
            "Phenotype": phen,
            "Node": node,
            "Type": ntype,
            "TotalEffect": float(total_effect),
            "Centrality": float(centrality_val),
            "TherapeuticScore": float(therapeutic_score)
        })

end_time = time.time()
print(f"\nFinished analysis in {end_time - start_time:.1f} s")

# -------------------------------
# PRINT TOP MICROBIAL & METABOLITE TARGET PER PHENOTYPE
# -------------------------------
final_targets = pd.DataFrame(all_targets)

if final_targets.empty:
    print("\n⚠️ No significant targets identified. Try lowering LASSO alpha or correlation threshold.")
else:
    phenotypes = final_targets["Phenotype"].unique()
    for phen in phenotypes:
        print(f"\n==================== {phen.upper()} ====================")
        phen_targets = final_targets[final_targets["Phenotype"] == phen]

        # Top Microbe
        top_microbe = phen_targets[phen_targets["Type"] == "microbe"].sort_values(
            by="TherapeuticScore", ascending=False
        ).head(1)
        if not top_microbe.empty:
            row = top_microbe.iloc[0]
            print(f"Top Microbe: {row['Node']}")
            print(f"  TotalEffect: {row['TotalEffect']:.4f}")
            print(f"  Centrality: {row['Centrality']:.4f}")
            print(f"  Therapeutic Importance Score (TIS): {row['TherapeuticScore']:.4f}")
        else:
            print("No microbial target identified.")

        # Top Metabolite
        top_metab = phen_targets[phen_targets["Type"] == "metabolite"].sort_values(
            by="TherapeuticScore", ascending=False
        ).head(1)
        if not top_metab.empty:
            row = top_metab.iloc[0]
            print(f"Top Metabolite: {row['Node']}")
            print(f"  TotalEffect: {row['TotalEffect']:.4f}")
            print(f"  Centrality: {row['Centrality']:.4f}")
            print(f"  Therapeutic Importance Score (TIS): {row['TherapeuticScore']:.4f}")
        else:
            print("No metabolite target identified.")

# -------------------------------
# FINAL OUTPUT — PRINT ALL TARGETS FOR EACH PHENOTYPE
# -------------------------------
if final_targets.empty:
    print("\n⚠️ No significant targets identified.")
else:
    phenotypes = final_targets["Phenotype"].unique()
    for phen in phenotypes:
        print(f"\n==================== {phen.upper()} (ALL) ====================")
        phen_targets = final_targets[final_targets["Phenotype"] == phen]

        # Microbes
        microbe_targets = phen_targets[phen_targets["Type"] == "microbe"].sort_values(
            by="TherapeuticScore", ascending=False
        )
        if not microbe_targets.empty:
            print("\n-- Microbial Targets --")
            for _, row in microbe_targets.iterrows():
                print(f"{row['Node']}: TotalEffect={row['TotalEffect']:.4f}, "
                      f"Centrality={row['Centrality']:.4f}, "
                      f"TIS={row['TherapeuticScore']:.4f}")
        else:
            print("\nNo microbial targets identified.")

        # Metabolites
        metab_targets = phen_targets[phen_targets["Type"] == "metabolite"].sort_values(
            by="TherapeuticScore", ascending=False
        )
        if not metab_targets.empty:
            print("\n-- Metabolite Targets --")
            for _, row in metab_targets.iterrows():
                print(f"{row['Node']}: TotalEffect={row['TotalEffect']:.4f}, "
                      f"Centrality={row['Centrality']:.4f}, "
                      f"TIS={row['TherapeuticScore']:.4f}")
        else:
            print("\nNo metabolite targets identified.")

# -------------------------------
# FINAL OUTPUT — PRINT STRONGEST TARGETS (OVERALL)
# -------------------------------
if final_targets.empty:
    print("\n⚠️ No significant targets identified.")
else:
    final_targets_sorted = final_targets.sort_values(by="TherapeuticScore", ascending=False)

    top_microbe = final_targets_sorted[final_targets_sorted["Type"] == "microbe"].head(1)
    top_metab = final_targets_sorted[final_targets_sorted["Type"] == "metabolite"].head(1)

    print("\n===== STRONGEST MICROBIAL THERAPEUTIC TARGET (OVERALL) =====")
    if not top_microbe.empty:
        m = top_microbe.iloc[0]
        print(f"Phenotype: {m['Phenotype']}")
        print(f"Node: {m['Node']}")
        print(f"Total Effect: {m['TotalEffect']:.4f}")
        print(f"Centrality: {m['Centrality']:.4f}")
        print(f"Therapeutic Importance Score: {m['TherapeuticScore']:.4f}")
    else:
        print("No microbial target identified.")

    print("\n===== STRONGEST METABOLITE THERAPEUTIC TARGET (OVERALL) =====")
    if not top_metab.empty:
        z = top_metab.iloc[0]
        print(f"Phenotype: {z['Phenotype']}")
        print(f"Node: {z['Node']}")
        print(f"Total Effect: {z['TotalEffect']:.4f}")
        print(f"Centrality: {z['Centrality']:.4f}")
        print(f"Therapeutic Importance Score: {z['TherapeuticScore']:.4f}")
    else:
        print("No metabolite target identified.")