Problem p00949 (C++) — buggy submission, verdict: Memory Limit Exceeded

#include <bits/stdc++.h>
using namespace std;
#define REP(i, n) for (int i = 0; i < (int)(n); ++i)
#define DEBUG(x) cerr << #x << " = " << x << endl
#define int long long
// int MOD = 2147483647LL;
int MOD = 2305266692540551LL;
int BASE = 4001;
signed main() {
  ios::sync_with_stdio(false);
  string S, T;
  cin >> S >> T;
  if (S.size() < T.size())
    swap(S, T);
  vector<vector<int>> u(T.size() + 1);
  int alpha[26];
  alpha[0] = 1;
  for (int i = 1; i < 26; ++i) {
    alpha[i] = (alpha[i - 1] * BASE) % MOD;
  }
  for (int l = 1; l <= (int)T.size(); ++l) {
    int hash = 0;
    REP(i, l) {
      hash += alpha[T[i] - 'a'];
      hash %= MOD;
    }
    u[l].push_back(hash);
    // cerr << T.substr(0, l) << " => " << hash << endl;
    for (int i = 1; i < (int)T.size() - l + 1; ++i) {
      hash -= alpha[T[i - 1] - 'a'];
      if (hash < 0)
        hash += MOD;
      hash += alpha[T[i - 1 + l] - 'a'];
      hash %= MOD;
      u[l].push_back(hash);
      // cerr << T.substr(i, l) << " => " << hash << endl;
    }
    sort(u[l].begin(), u[l].end());
  }
  for (int l = (int)T.size(); l >= 1; --l) {
    int hash = 0;
    REP(i, l) {
      hash += alpha[S[i] - 'a'];
      hash %= MOD;
    }
    if (binary_search(u[l].begin(), u[l].end(), hash)) {
      cout << l << endl;
      return 0;
    }
    // cerr << S.substr(0, l) << " => " << hash << endl;
    for (int i = 1; i < (int)S.size() - l + 1; ++i) {
      hash -= alpha[S[i - 1] - 'a'];
      if (hash < 0)
        hash += MOD;
      hash += alpha[S[i - 1 + l] - 'a'];
      hash %= MOD;
      // cerr << S.substr(i, l) << " => " << hash << endl;
      if (binary_search(u[l].begin(), u[l].end(), hash)) {
        cout << l << endl;
        return 0;
      }
    }
  }
  cout << 0 << endl;
  return 0;
}

Error: MLE

Fixed submission:

#include <bits/stdc++.h>
using namespace std;
#define REP(i, n) for (int i = 0; i < (int)(n); ++i)
#define DEBUG(x) cerr << #x << " = " << x << endl
#define int long long
// int MOD = 2147483647LL;
int MOD = 2305266692540551LL;
int BASE = 4001;
signed main() {
  ios::sync_with_stdio(false);
  string S, T;
  cin >> S >> T;
  if (S.size() < T.size())
    swap(S, T);
  vector<set<int>> u(T.size() + 1);
  int alpha[26];
  alpha[0] = 1;
  for (int i = 1; i < 26; ++i) {
    alpha[i] = (alpha[i - 1] * BASE) % MOD;
  }
  for (int l = 1; l <= (int)T.size(); ++l) {
    int hash = 0;
    REP(i, l) {
      hash += alpha[T[i] - 'a'];
      hash %= MOD;
    }
    u[l].insert(hash);
    // cerr << T.substr(0, l) << " => " << hash << endl;
    for (int i = 1; i < (int)T.size() - l + 1; ++i) {
      hash -= alpha[T[i - 1] - 'a'];
      if (hash < 0)
        hash += MOD;
      hash += alpha[T[i - 1 + l] - 'a'];
      hash %= MOD;
      u[l].insert(hash);
      // cerr << T.substr(i, l) << " => " << hash << endl;
    }
  }
  for (int l = (int)T.size(); l >= 1; --l) {
    int hash = 0;
    REP(i, l) {
      hash += alpha[S[i] - 'a'];
      hash %= MOD;
    }
    if (u[l].count(hash)) {
      cout << l << endl;
      return 0;
    }
    // cerr << S.substr(0, l) << " => " << hash << endl;
    for (int i = 1; i < (int)S.size() - l + 1; ++i) {
      hash -= alpha[S[i - 1] - 'a'];
      if (hash < 0)
        hash += MOD;
      hash += alpha[S[i - 1 + l] - 'a'];
      hash %= MOD;
      // cerr << S.substr(i, l) << " => " << hash << endl;
      if (u[l].count(hash)) {
        cout << l << endl;
        return 0;
      }
    }
  }
  cout << 0 << endl;
  return 0;
}