Task: Determine if two triangles overlap
C: #include <errno.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>

typedef struct {
    double x, y;
} Point;

double det2D(const Point * const p1, const Point * const p2, const Point * const p3) {
    return p1->x * (p2->y - p3->y)
        + p2->x * (p3->y - p1->y)
        + p3->x * (p1->y - p2->y);
}

void checkTriWinding(Point * p1, Point * p2, Point * p3, bool allowReversed) {
    double detTri = det2D(p1, p2, p3);
    if (detTri < 0.0) {
        if (allowReversed) {
            double t = p3->x;
            p3->x = p2->x;
            p2->x = t;

            t = p3->y;
            p3->y = p2->y;
            p2->y = t;
        } else {
            errno = 1;
        }
    }
}

bool boundaryCollideChk(const Point *p1, const Point *p2, const Point *p3, double eps) {
    return det2D(p1, p2, p3) < eps;
}

bool boundaryDoesntCollideChk(const Point *p1, const Point *p2, const Point *p3, double eps) {
    return det2D(p1, p2, p3) <= eps;
}

bool triTri2D(Point t1[], Point t2[], double eps, bool allowReversed, bool onBoundary) {
    bool(*chkEdge)(Point*, Point*, Point*, double);
    int i;

    
    checkTriWinding(&t1[0], &t1[1], &t1[2], allowReversed);
    if (errno != 0) {
        return false;
    }
    checkTriWinding(&t2[0], &t2[1], &t2[2], allowReversed);
    if (errno != 0) {
        return false;
    }

    if (onBoundary) {
        
        chkEdge = boundaryCollideChk;
    } else {
        
        chkEdge = boundaryDoesntCollideChk;
    }

    
    for (i = 0; i < 3; ++i) {
        int j = (i + 1) % 3;

        
        
        if (chkEdge(&t1[i], &t1[j], &t2[0], eps) &&
            chkEdge(&t1[i], &t1[j], &t2[1], eps) &&
            chkEdge(&t1[i], &t1[j], &t2[2], eps)) {
            return false;
        }
    }

    
    for (i = 0; i < 3; i++) {
        int j = (i + 1) % 3;

        
        
        if (chkEdge(&t2[i], &t2[j], &t1[0], eps) &&
            chkEdge(&t2[i], &t2[j], &t1[1], eps) &&
            chkEdge(&t2[i], &t2[j], &t1[2], eps))
            return false;
    }

    
    return true;
}

int main() {
    {
        Point t1[] = { {0, 0}, {5, 0}, {0, 5} };
        Point t2[] = { {0, 0}, {5, 0}, {0, 6} };
        printf("%d,true\n", triTri2D(t1, t2, 0.0, false, true));
    }

    {
        Point t1[] = { {0, 0}, {0, 5}, {5, 0} };
        Point t2[] = { {0, 0}, {0, 5}, {5, 0} };
        printf("%d,true\n", triTri2D(t1, t2, 0.0, true, true));
    }

    {
        Point t1[] = { {0, 0}, {5, 0}, {0, 5} };
        Point t2[] = { {-10, 0}, {-5, 0}, {-1, 6} };
        printf("%d,false\n", triTri2D(t1, t2, 0.0, false, true));
    }

    {
        Point t1[] = { {0, 0}, {5, 0}, {2.5, 5} };
        Point t2[] = { {0, 4}, {2.5, -1}, {5, 4} };
        printf("%d,true\n", triTri2D(t1, t2, 0.0, false, true));
    }

    {
        Point t1[] = { {0, 0}, {1, 1}, {0, 2} };
        Point t2[] = { {2, 1}, {3, 0}, {3, 2} };
        printf("%d,false\n", triTri2D(t1, t2, 0.0, false, true));
    }

    {
        Point t1[] = { {0, 0}, {1, 1}, {0, 2} };
        Point t2[] = { {2, 1}, {3, -2}, {3, 4} };
        printf("%d,false\n", triTri2D(t1, t2, 0.0, false, true));
    }

    
    {
        Point t1[] = { {0, 0}, {1, 0}, {0, 1} };
        Point t2[] = { {1, 0}, {2, 0}, {1, 1} };
        printf("%d,true\n", triTri2D(t1, t2, 0.0, false, true));
    }

    
    {
        Point t1[] = { {0, 0}, {1, 0}, {0, 1} };
        Point t2[] = { {1, 0}, {2, 0}, {1, 1} };
        printf("%d,false\n", triTri2D(t1, t2, 0.0, false, false));
    }

    return EXIT_SUCCESS;
}

Java: import java.util.function.BiFunction;

public class TriangleOverlap {
    private static class Pair {
        double first;
        double second;

        Pair(double first, double second) {
            this.first = first;
            this.second = second;
        }

        @Override
        public String toString() {
            return String.format("(%s, %s)", first, second);
        }
    }

    private static class Triangle {
        Pair p1, p2, p3;

        Triangle(Pair p1, Pair p2, Pair p3) {
            this.p1 = p1;
            this.p2 = p2;
            this.p3 = p3;
        }

        @Override
        public String toString() {
            return String.format("Triangle: %s, %s, %s", p1, p2, p3);
        }
    }

    private static double det2D(Triangle t) {
        Pair p1 = t.p1;
        Pair p2 = t.p2;
        Pair p3 = t.p3;
        return p1.first * (p2.second - p3.second)
            + p2.first * (p3.second - p1.second)
            + p3.first * (p1.second - p2.second);
    }

    private static void checkTriWinding(Triangle t, boolean allowReversed) {
        double detTri = det2D(t);
        if (detTri < 0.0) {
            if (allowReversed) {
                Pair a = t.p3;
                t.p3 = t.p2;
                t.p2 = a;
            } else throw new RuntimeException("Triangle has wrong winding direction");
        }
    }

    private static boolean boundaryCollideChk(Triangle t, double eps) {
        return det2D(t) < eps;
    }

    private static boolean boundaryDoesntCollideChk(Triangle t, double eps) {
        return det2D(t) <= eps;
    }

    private static boolean triTri2D(Triangle t1, Triangle t2) {
        return triTri2D(t1, t2, 0.0, false, true);
    }

    private static boolean triTri2D(Triangle t1, Triangle t2, double eps, boolean allowedReversed) {
        return triTri2D(t1, t2, eps, allowedReversed, true);
    }

    private static boolean triTri2D(Triangle t1, Triangle t2, double eps, boolean allowedReversed, boolean onBoundary) {
        
        checkTriWinding(t1, allowedReversed);
        checkTriWinding(t2, allowedReversed);
        
        BiFunction<Triangle, Double, Boolean> chkEdge = onBoundary ? TriangleOverlap::boundaryCollideChk : TriangleOverlap::boundaryDoesntCollideChk;
        Pair[] lp1 = new Pair[]{t1.p1, t1.p2, t1.p3};
        Pair[] lp2 = new Pair[]{t2.p1, t2.p2, t2.p3};

        
        for (int i = 0; i < 3; ++i) {
            int j = (i + 1) % 3;
            
            
            if (chkEdge.apply(new Triangle(lp1[i], lp1[j], lp2[0]), eps) &&
                chkEdge.apply(new Triangle(lp1[i], lp1[j], lp2[1]), eps) &&
                chkEdge.apply(new Triangle(lp1[i], lp1[j], lp2[2]), eps)) return false;
        }

        
        for (int i = 0; i < 3; ++i) {
            int j = (i + 1) % 3;
            
            
            if (chkEdge.apply(new Triangle(lp2[i], lp2[j], lp1[0]), eps) &&
                chkEdge.apply(new Triangle(lp2[i], lp2[j], lp1[1]), eps) &&
                chkEdge.apply(new Triangle(lp2[i], lp2[j], lp1[2]), eps)) return false;
        }

        
        return true;
    }

    public static void main(String[] args) {
        Triangle t1 = new Triangle(new Pair(0.0, 0.0), new Pair(5.0, 0.0), new Pair(0.0, 5.0));
        Triangle t2 = new Triangle(new Pair(0.0, 0.0), new Pair(5.0, 0.0), new Pair(0.0, 6.0));
        System.out.printf("%s and\n%s\n", t1, t2);
        if (triTri2D(t1, t2)) {
            System.out.println("overlap");
        } else {
            System.out.println("do not overlap");
        }

        
        t1 = new Triangle(new Pair(0.0, 0.0), new Pair(0.0, 5.0), new Pair(5.0, 0.0));
        t2 = t1;
        System.out.printf("\n%s and\n%s\n", t1, t2);
        if (triTri2D(t1, t2, 0.0, true)) {
            System.out.println("overlap (reversed)");
        } else {
            System.out.println("do not overlap");
        }

        t1 = new Triangle(new Pair(0.0, 0.0), new Pair(5.0, 0.0), new Pair(0.0, 5.0));
        t2 = new Triangle(new Pair(-10.0, 0.0), new Pair(-5.0, 0.0), new Pair(-1.0, 6.0));
        System.out.printf("\n%s and\n%s\n", t1, t2);
        if (triTri2D(t1, t2)) {
            System.out.println("overlap");
        } else {
            System.out.println("do not overlap");
        }

        t1.p3 = new Pair(2.5, 5.0);
        t2 = new Triangle(new Pair(0.0, 4.0), new Pair(2.5, -1.0), new Pair(5.0, 4.0));
        System.out.printf("\n%s and\n%s\n", t1, t2);
        if (triTri2D(t1, t2)) {
            System.out.println("overlap");
        } else {
            System.out.println("do not overlap");
        }

        t1 = new Triangle(new Pair(0.0, 0.0), new Pair(1.0, 1.0), new Pair(0.0, 2.0));
        t2 = new Triangle(new Pair(2.0, 1.0), new Pair(3.0, 0.0), new Pair(3.0, 2.0));
        System.out.printf("\n%s and\n%s\n", t1, t2);
        if (triTri2D(t1, t2)) {
            System.out.println("overlap");
        } else {
            System.out.println("do not overlap");
        }

        t2 = new Triangle(new Pair(2.0, 1.0), new Pair(3.0, -2.0), new Pair(3.0, 4.0));
        System.out.printf("\n%s and\n%s\n", t1, t2);
        if (triTri2D(t1, t2)) {
            System.out.println("overlap");
        } else {
            System.out.println("do not overlap");
        }

        t1 = new Triangle(new Pair(0.0, 0.0), new Pair(1.0, 0.0), new Pair(0.0, 1.0));
        t2 = new Triangle(new Pair(1.0, 0.0), new Pair(2.0, 0.0), new Pair(1.0, 1.1));
        System.out.printf("\n%s and\n%s\n", t1, t2);
        System.out.println("which have only a single corner in contact, if boundary points collide");
        if (triTri2D(t1, t2)) {
            System.out.println("overlap");
        } else {
            System.out.println("do not overlap");
        }

        System.out.printf("\n%s and\n%s\n", t1, t2);
        System.out.println("which have only a single corner in contact, if boundary points do not collide");
        if (triTri2D(t1, t2, 0.0, false, false)) {
            System.out.println("overlap");
        } else {
            System.out.println("do not overlap");
        }
    }
}