Upload 8 files

app.py

@@ -0,0 +1,117 @@
+from flask import Flask, render_template, request, redirect, url_for
+import pandas as pd
+from fuzzywuzzy import fuzz
+import os
+from flask import send_file
+
+app = Flask(__name__)
+app.config['UPLOAD_FOLDER'] = 'uploads'
+app.config['OUTPUT_FOLDER'] = 'output'
+output_file = None
+
+def process_csv(input_path):
+    global output_file
+    df = pd.read_csv(input_path)
+
+    df['Address'] = df['Address (street)'].astype(str) + '-' + df['Postal code'].astype(str) + '-' + df['City'].astype(
+        str) + '-' + df['Country'].astype(str) + df['Region'].astype(str)
+    # df['Name'] = df['NAME_FIRST'].astype(str) + '-' + df['NAME_LAST'].astype(str) + '-' + df['NAME3'].astype(str) + '-' + df['NAME4'].astype(str)
+    df['Name'] = df['Vendor Name'].astype(str)
+
+    df['Name'] = df['Name'].str.lower()
+    df['Address'] = df['Address'].str.lower()
+
+    df.sort_values(['Name'], inplace=True)
+    df = df.reset_index(drop=True)
+
+    df['name_fuzzy_ratio'] = ''
+    df['address_fuzzy_ratio'] = ''
+    df['name_based_group'] = ''
+    df['address_based_group'] = ''
+
+    last_row_index = len(df) - 1
+    df.at[0, 'name_fuzzy_ratio'] = 100
+    df.at[0, 'address_fuzzy_ratio'] = 100
+    df.at[last_row_index, 'name_fuzzy_ratio'] = 100
+    df.at[last_row_index, 'address_fuzzy_ratio'] = 100
+
+    for i in range(1, last_row_index):
+        current_name = df['Name'].iloc[i]
+        previous_name = df['Name'].iloc[i - 1]
+        fuzzy_ratio = fuzz.ratio(previous_name, current_name)
+        df.at[i, 'name_fuzzy_ratio'] = fuzzy_ratio
+
+    df['name_fuzzy_ratio'] = pd.to_numeric(df['name_fuzzy_ratio'], errors='coerce')
+
+    group_counter = 1
+    df.at[0, 'name_based_group'] = group_counter
+
+    for i in range(1, len(df)):
+        if df.at[i, 'name_fuzzy_ratio'] > 80:
+            df.at[i, 'name_based_group'] = df.at[i - 1, 'name_based_group']
+        else:
+            group_counter += 1
+            df.at[i, 'name_based_group'] = group_counter
+
+    group = df.at[0, 'name_based_group']
+
+    df.sort_values(['name_based_group', 'Address'], inplace=True)
+    df = df.reset_index(drop=True)
+
+    for i in range(1, last_row_index):
+        current_address = df['Address'].iloc[i]
+        previous_address = df['Address'].iloc[i - 1]
+        fuzzy_ratio = fuzz.ratio(previous_address, current_address)
+        df.at[i, 'address_fuzzy_ratio'] = fuzzy_ratio
+
+    df['address_fuzzy_ratio'] = pd.to_numeric(df['address_fuzzy_ratio'], errors='coerce')
+
+    address_group_counter = 1
+    df.at[0, 'address_based_group'] = str(address_group_counter)
+
+    for i in range(1, len(df)):
+        if df.at[i, 'address_fuzzy_ratio'] > 70:
+            df.at[i, 'address_based_group'] = df.at[i - 1, 'address_based_group']
+        else:
+            if df.at[i, 'name_based_group'] != group:
+                address_group_counter = 1
+                group = df.at[i, 'name_based_group']
+            else:
+                address_group_counter += 1
+            df.at[i, 'address_based_group'] = str(address_group_counter)
+
+    # Concatenate for unique group name
+    df['Group'] = df.apply(lambda row: 'Group_{}_{}'.format(row['name_based_group'], row['address_based_group']),
+                           axis=1)
+
+    columns_to_drop = ['name_fuzzy_ratio', 'address_fuzzy_ratio', 'Address', 'Name']
+    df.drop(columns=columns_to_drop, inplace=True)
+
+    output_path = os.path.join(app.config['OUTPUT_FOLDER'], 'output.csv')
+    df.to_csv(output_path, index=False)
+    output_file = 'output.csv'
+    return output_path
+
+@app.route('/', methods=['GET', 'POST'])
+def upload_file():
+    global output_file
+    if request.method == 'POST':
+        file = request.files['file']
+        if file:
+            file_path = os.path.join(app.config['UPLOAD_FOLDER'], file.filename)
+            file.save(file_path)
+            output_file = process_csv(file_path)
+            return redirect(url_for('upload_file'))
+
+    return render_template('index.html', output_file=output_file)
+
+@app.route('/downloads/output.csv')
+def download_file():
+    output_file = os.path.join(app.config['OUTPUT_FOLDER'], 'output.csv')
+    return send_file(output_file, as_attachment=True)
+
+
+if __name__ == '__main__':
+    app.run(debug=True)
+
+

duplicate.yaml

@@ -0,0 +1,46 @@
+name: duplicate_removal
+channels:
+  - anaconda
+  - defaults
+dependencies:
+  - blas=1.0=mkl
+  - bottleneck=1.3.7=py310h9128911_0
+  - bzip2=1.0.8=h2bbff1b_5
+  - ca-certificates=2023.12.12=haa95532_0
+  - click=8.1.7=py310haa95532_0
+  - colorama=0.4.6=py310haa95532_0
+  - flask=2.2.2=py310haa95532_0
+  - fuzzywuzzy=0.18.0=py310haa95532_0
+  - intel-openmp=2023.1.0=h59b6b97_46320
+  - itsdangerous=2.0.1=pyhd3eb1b0_0
+  - jinja2=3.1.3=py310haa95532_0
+  - libffi=3.4.4=hd77b12b_0
+  - markupsafe=2.1.3=py310h2bbff1b_0
+  - mkl=2023.1.0=h6b88ed4_46358
+  - mkl-service=2.4.0=py310h2bbff1b_1
+  - mkl_fft=1.3.8=py310h2bbff1b_0
+  - mkl_random=1.2.4=py310h59b6b97_0
+  - numexpr=2.8.7=py310h2cd9be0_0
+  - numpy=1.26.4=py310h055cbcc_0
+  - numpy-base=1.26.4=py310h65a83cf_0
+  - openssl=3.0.13=h2bbff1b_0
+  - pandas=2.2.1=py310h5da7b33_0
+  - pip=23.3.1=py310haa95532_0
+  - python=3.10.13=he1021f5_0
+  - python-dateutil=2.8.2=pyhd3eb1b0_0
+  - python-levenshtein=0.12.2=py310h2bbff1b_0
+  - python-tzdata=2023.3=pyhd3eb1b0_0
+  - pytz=2023.3.post1=py310haa95532_0
+  - setuptools=68.2.2=py310haa95532_0
+  - six=1.16.0=pyhd3eb1b0_1
+  - sqlite=3.41.2=h2bbff1b_0
+  - tbb=2021.8.0=h59b6b97_0
+  - tk=8.6.12=h2bbff1b_0
+  - tzdata=2024a=h04d1e81_0
+  - vc=14.2=h21ff451_1
+  - vs2015_runtime=14.27.29016=h5e58377_2
+  - werkzeug=2.3.8=py310haa95532_0
+  - wheel=0.41.2=py310haa95532_0
+  - xz=5.4.6=h8cc25b3_0
+  - zlib=1.2.13=h8cc25b3_0
+prefix: C:\Users\snigd\.conda\envs\duplicate_removal

static/script.js

@@ -0,0 +1,28 @@
+function submitForm() {
+    var fileInput = document.getElementById('csvFile');
+    var processingMsg = document.getElementById('processingMsg');
+
+    if (fileInput.files.length === 0) {
+        alert('Please select a CSV file.');
+        return;
+    }
+
+    var formData = new FormData();
+    formData.append('csvFile', fileInput.files[0]);
+
+    // Show processing message
+    document.getElementById('uploadForm').classList.add('hidden');
+    processingMsg.classList.remove('hidden');
+
+    // Simulate backend processing (replace with actual AJAX call)
+    setTimeout(function() {
+        // After processing (simulated with setTimeout), show success message
+        processingMsg.innerHTML = '<p>File processed successfully. <a href="#" onclick="downloadProcessedFile()">Download processed file</a></p>';
+    }, 2000);
+}
+
+function downloadProcessedFile() {
+    // Here you can add code to download the processed file
+    alert('Downloading processed file...');
+    // Replace this alert with your actual download logic
+}

static/styles.css

@@ -0,0 +1,54 @@
+body {
+    font-family: Arial, sans-serif;
+    background-color: #f0f0f0;
+    margin: 0;
+    padding: 200px;
+}
+
+.container {
+    max-width: 600px;
+    margin: 0 auto;
+    align-items: center;
+    background-color: #fff;
+    padding: 20px;
+    border-radius: 5px;
+    box-shadow: 0 2px 5px rgba(0, 0, 0, 0.1);
+}
+
+.btn {
+    padding: 10px 200px;
+}
+
+h1 {
+    text-align: center;
+    color: #333;
+}
+
+form {
+    display: flex;
+    flex-direction: column;
+}
+
+input[type="file"] {
+    margin-bottom: 10px;
+}
+
+button {
+    padding: 10px 20px;
+    background-color: #007bff;
+    color: #fff;
+    border: none;
+    cursor: pointer;
+}
+
+button:hover {
+    background-color: #0056b3;
+}
+
+.hidden {
+    display: none;
+}
+
+#processingMsg {
+    text-align: center;
+}

templates/index.html

@@ -0,0 +1,29 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Duplicate Removal</title>
+    <link rel="stylesheet" href="{{ url_for('static', filename='styles.css') }}">
+</head>
+<body>
+    <div class="container">
+        <h1>Duplicate Removal</h1>
+        <form id="uploadForm" enctype="multipart/form-data" action="/" method="POST">
+            <input type="file" name="file" id="csvFile" accept=".csv">
+            <button type="submit" id="submitBtn">Submit</button>
+        </form>
+        <div id="processingMsg" class="hidden">
+            <p>Processing...</p>
+        </div>
+        {% if output_file %}
+        <div class="btn">
+            <a href="{{ url_for('download_file', filename=output_file) }}" download>
+                <button>Download Processed CSV</button>
+            </a>
+        </div>
+        {% endif %}
+    </div>
+    <script src="{{ url_for('static', filename='script.js') }}"></script>
+</body>
+</html>