Problem p00781 (C++) — buggy submission, verdict: Memory Limit Exceeded

#include <algorithm>
#include <iostream>
#include <set>
#include <vector>
using namespace std;

long long t[1 << 20];

int mirror(int comb) {
  int res = 0;
  for (int i = 0; i < 5; i++) {
    for (int j = 0; j < 5; j++) {
      res |= (((comb >> (i * 5 + j))) & 1) << (i * 5 + 4 - j);
    }
  }
  return res;
}

int rot(int comb) {
  int res = 0;
  for (int i = 0; i < 5; i++) {
    for (int j = 0; j < 5; j++) {
      if ((comb >> (i * 5 + j)) & 1) {
        res |= 1 << ((4 - j) * 5 + i);
      }
    }
  }
  return res;
}

void print(int t) {
  for (int i = 0; i < 5; i++) {
    for (int j = 0; j < 5; j++)
      cout << ((t >> ((i * 5) + j)) & 1);
    cout << endl;
  }
  cout << endl;
}

int main() {
  string s[10];
  int mask = (1 << 25) - 1;
  set<int> sa, sb;
  vector<int> a, b;

  while (cin >> s[0], s[0] != "END") {
    for (int i = 1; i < 10; i++)
      cin >> s[i];
    for (int i = 0; i < (1 << 20); i++)
      t[i] = 0LL;

    int ans = 0;
    int comb = (1 << 5) - 1;
    while (comb < (1 << 10)) {
      a.clear();
      b.clear();
      for (int i = 0; i < 10; i++) {
        if ((comb >> i) & 1)
          a.push_back(i);
        else
          b.push_back(i);
      }
      sa.clear();
      sb.clear();
      do {
        for (int k = 0; k < (1 << 5); k++) {
          int tmp = 0;
          for (int i = 0; i < 5; i++) {
            for (int j = 0; j < 5; j++) {
              if (s[a[i]][j] == '1') {
                if ((k >> i) & 1)
                  tmp |= 1 << ((i * 5) + j);
                else
                  tmp |= 1 << ((i * 5) + 4 - j);
              }
            }
          }
          sa.insert(tmp);
        }
      } while (next_permutation(a.begin(), a.end()));

      do {
        for (int k = 0; k < (1 << 5); k++) {
          int tmp = 0;
          for (int i = 0; i < 5; i++) {
            for (int j = 0; j < 5; j++) {
              if (s[b[i]][j] == '1') {
                if ((k >> i) & 1)
                  tmp |= 1 << ((j * 5) + i);
                else
                  tmp |= 1 << (((4 - j) * 5) + i);
              }
            }
          }
          sb.insert(tmp ^ mask);
        }
      } while (next_permutation(b.begin(), b.end()));

      for (set<int>::iterator ite = sa.begin(); ite != sa.end(); ite++) {
        int x = *ite;
        if (sb.find(x) != sb.end()) {
          long long f = 0LL;
          for (int j = 0; j < 2; j++) {
            x = mirror(x);
            for (int k = 0; k < 4; k++) {
              x = rot(x);
              if (k & 1)
                f += (t[x / 64] >> (x % 64)) & 1LL;
              else
                f += (t[(x ^ mask) / 64] >> ((x ^ mask) % 64)) & 1LL;
            }
          }
          if (!f) {
            t[x / 64] |= 1LL << (x % 64);
            ans++;
          }
        }
      }

      int x = comb & -comb, y = comb + x;
      comb = (((comb & ~y) / x) >> 1) | y;
    }
    cout << ans << endl;
  }
}

Error: MLE

Fixed submission:

#include <algorithm>
#include <iostream>
#include <set>
#include <vector>
using namespace std;

bool t[1 << 25];

int mirror(int comb) {
  int res = 0;
  for (int i = 0; i < 5; i++) {
    for (int j = 0; j < 5; j++) {
      res |= (((comb >> (i * 5 + j))) & 1) << (i * 5 + 4 - j);
    }
  }
  return res;
}

int rot(int comb) {
  int res = 0;
  for (int i = 0; i < 5; i++) {
    for (int j = 0; j < 5; j++) {
      if ((comb >> (i * 5 + j)) & 1) {
        res |= 1 << ((4 - j) * 5 + i);
      }
    }
  }
  return res;
}

void print(int t) {
  for (int i = 0; i < 5; i++) {
    for (int j = 0; j < 5; j++)
      cout << ((t >> ((i * 5) + j)) & 1);
    cout << endl;
  }
  cout << endl;
}
int main() {
  string s[10];
  int mask = (1 << 25) - 1;
  set<int> sa, sb;

  while (cin >> s[0], s[0] != "END") {
    for (int i = 1; i < 10; i++)
      cin >> s[i];
    for (int i = 0; i < (1 << 25); i++)
      t[i] = false;

    int ans = 0;
    int comb = (1 << 5) - 1;
    while (comb < (1 << 10)) {
      vector<int> a, b;
      for (int i = 0; i < 10; i++) {
        if ((comb >> i) & 1)
          a.push_back(i);
        else
          b.push_back(i);
      }
      sa.clear(), sb.clear();
      do {
        for (int k = 0; k < (1 << 5); k++) {
          int tmp = 0;
          for (int i = 0; i < 5; i++) {
            for (int j = 0; j < 5; j++) {
              if (s[a[i]][j] == '1') {
                if ((k >> i) & 1)
                  tmp |= 1 << ((i * 5) + j);
                else
                  tmp |= 1 << ((i * 5) + 4 - j);
              }
            }
          }
          sa.insert(tmp);
        }
      } while (next_permutation(a.begin(), a.end()));

      do {
        for (int k = 0; k < (1 << 5); k++) {
          int tmp = 0;
          for (int i = 0; i < 5; i++) {
            for (int j = 0; j < 5; j++) {
              if (s[b[i]][j] == '1') {
                if ((k >> i) & 1)
                  tmp |= 1 << ((j * 5) + i);
                else
                  tmp |= 1 << (((4 - j) * 5) + i);
              }
            }
          }
          sb.insert(tmp ^ mask);
        }
      } while (next_permutation(b.begin(), b.end()));

      for (set<int>::iterator ite = sa.begin(); ite != sa.end(); ite++) {
        int x = *ite;
        if (sb.find(x) != sb.end()) {
          bool f = false;
          for (int j = 0; j < 2; j++) {
            x = mirror(x);
            for (int k = 0; k < 4; k++) {
              x = rot(x);
              if (k & 1)
                f |= t[x];
              else
                f |= t[x ^ mask];
            }
          }
          if (!f) {
            t[x] = true;
            ans++;
          }
        }
      }

      int x = comb & -comb, y = comb + x;
      comb = (((comb & ~y) / x) >> 1) | y;
    }
    cout << ans << endl;
  }
}