semeru/Code-Code-CloneDetection-BigCloneBench huggingface dataset:
public void retrieveFiles() throws DataSyncException {
        try {
            ftp.connect(hostname, port);
            boolean success = ftp.login(username, password);
            log.info("FTP Login:" + success);
            if (success) {
                System.out.println(directory);
                ftp.changeWorkingDirectory(directory);
                ftp.setFileType(FTP.ASCII_FILE_TYPE);
                ftp.enterLocalPassiveMode();
                ftp.setRemoteVerificationEnabled(false);
                FTPFile[] files = ftp.listFiles();
                for (FTPFile file : files) {
                    ftp.setFileType(file.getType());
                    log.debug(file.getName() + "," + file.getSize());
                    FileOutputStream output = new FileOutputStream(localDirectory + file.getName());
                    try {
                        ftp.retrieveFile(file.getName(), output);
                    } finally {
                        IOUtils.closeQuietly(output);
                    }
                }
            }
        } catch (Exception e) {
            throw new DataSyncException(e);
        } finally {
            try {
                ftp.disconnect();
            } catch (IOException e) {
            }
        }
    }
@Override
    public void save(String arxivId, InputStream inputStream, String encoding) {
        String filename = StringUtil.arxivid2filename(arxivId, "tex");
        try {
            Writer writer = new OutputStreamWriter(new FileOutputStream(String.format("%s/%s", LATEX_DOCUMENT_DIR, filename)), encoding);
            IOUtils.copy(inputStream, writer, encoding);
            writer.flush();
            writer.close();
            inputStream.close();
        } catch (IOException e) {
            logger.error("Failed to save the Latex source with id='{}'", arxivId, e);
            throw new RuntimeException(e);
        }
    }
public static void copy(File source, File sink) throws IOException {
        if (source == null) throw new NullPointerException("Source file must not be null");
        if (sink == null) throw new NullPointerException("Target file must not be null");
        if (!source.exists()) throw new IOException("Source file " + source.getPath() + " does not exist");
        if (!source.isFile()) throw new IOException("Source file " + source.getPath() + " is not a regular file");
        if (!source.canRead()) throw new IOException("Source file " + source.getPath() + " can not be read (missing acces right)");
        if (!sink.exists()) throw new IOException("Target file " + sink.getPath() + " does not exist");
        if (!sink.isFile()) throw new IOException("Target file " + sink.getPath() + " is not a regular file");
        if (!sink.canWrite()) throw new IOException("Target file " + sink.getPath() + " is write protected");
        InputStream input = null;
        OutputStream output = null;
        try {
            input = new FileInputStream(source);
            output = new FileOutputStream(sink);
            byte[] buffer = new byte[1024];
            while (input.available() > 0) {
                int bread = input.read(buffer);
                if (bread > 0) output.write(buffer, 0, bread);
            }
        } finally {
            if (input != null) try {
                input.close();
            } catch (IOException x) {
            }
            if (output != null) try {
                output.close();
            } catch (IOException x) {
            }
        }
    }
public void copyHashAllFilesToDirectory(String baseDirStr, Hashtable newNamesTable, String destDirStr) throws Exception {
        if (baseDirStr.endsWith(sep)) {
            baseDirStr = baseDirStr.substring(0, baseDirStr.length() - 1);
        }
        if (destDirStr.endsWith(sep)) {
            destDirStr = destDirStr.substring(0, destDirStr.length() - 1);
        }
        FileUtils.getInstance().createDirectory(baseDirStr);
        if (null == newNamesTable) {
            newNamesTable = new Hashtable();
        }
        BufferedInputStream in = null;
        BufferedOutputStream out = null;
        byte dataBuff[] = new byte[bufferSize];
        File baseDir = new File(baseDirStr);
        baseDir.mkdirs();
        if ((baseDir.exists()) && (baseDir.isDirectory())) {
            if (!newNamesTable.isEmpty()) {
                Enumeration enumFiles = newNamesTable.keys();
                while (enumFiles.hasMoreElements()) {
                    String newName = (String) enumFiles.nextElement();
                    String oldPathName = (String) newNamesTable.get(newName);
                    if ((newName != null) && (!"".equals(newName)) && (oldPathName != null) && (!"".equals(oldPathName))) {
                        String newPathFileName = destDirStr + sep + newName;
                        String oldPathFileName = baseDirStr + sep + oldPathName;
                        if (oldPathName.startsWith(sep)) {
                            oldPathFileName = baseDirStr + oldPathName;
                        }
                        File f = new File(oldPathFileName);
                        if ((f.exists()) && (f.isFile())) {
                            in = new BufferedInputStream(new FileInputStream(oldPathFileName), bufferSize);
                            out = new BufferedOutputStream(new FileOutputStream(newPathFileName), bufferSize);
                            int readLen;
                            while ((readLen = in.read(dataBuff)) > 0) {
                                out.write(dataBuff, 0, readLen);
                            }
                            out.flush();
                            in.close();
                            out.close();
                        } else {
                        }
                    }
                }
            } else {
            }
        } else {
            throw new Exception("Base (baseDirStr) dir not exist !");
        }
    }
public long copyDirAllFilesToDirectory(String baseDirStr, String destDirStr) throws Exception {
        long plussQuotaSize = 0;
        if (baseDirStr.endsWith(sep)) {
            baseDirStr = baseDirStr.substring(0, baseDirStr.length() - 1);
        }
        if (destDirStr.endsWith(sep)) {
            destDirStr = destDirStr.substring(0, destDirStr.length() - 1);
        }
        FileUtils.getInstance().createDirectory(destDirStr);
        BufferedInputStream in = null;
        BufferedOutputStream out = null;
        byte dataBuff[] = new byte[bufferSize];
        File baseDir = new File(baseDirStr);
        baseDir.mkdirs();
        if (!baseDir.exists()) {
            createDirectory(baseDirStr);
        }
        if ((baseDir.exists()) && (baseDir.isDirectory())) {
            String[] entryList = baseDir.list();
            if (entryList.length > 0) {
                for (int pos = 0; pos < entryList.length; pos++) {
                    String entryName = entryList[pos];
                    String oldPathFileName = baseDirStr + sep + entryName;
                    File entryFile = new File(oldPathFileName);
                    if (entryFile.isFile()) {
                        String newPathFileName = destDirStr + sep + entryName;
                        File newFile = new File(newPathFileName);
                        if (newFile.exists()) {
                            plussQuotaSize -= newFile.length();
                            newFile.delete();
                        }
                        in = new BufferedInputStream(new FileInputStream(oldPathFileName), bufferSize);
                        out = new BufferedOutputStream(new FileOutputStream(newPathFileName), bufferSize);
                        int readLen;
                        while ((readLen = in.read(dataBuff)) > 0) {
                            out.write(dataBuff, 0, readLen);
                            plussQuotaSize += readLen;
                        }
                        out.flush();
                        in.close();
                        out.close();
                    }
                }
            }
        } else {
            throw new Exception("Base dir not exist ! baseDirStr = (" + baseDirStr + ")");
        }
        return plussQuotaSize;
    }
public long copyDirAllFilesToDirectoryRecursive(String baseDirStr, String destDirStr, boolean copyOutputsRtIDsDirs) throws Exception {
        long plussQuotaSize = 0;
        if (baseDirStr.endsWith(sep)) {
            baseDirStr = baseDirStr.substring(0, baseDirStr.length() - 1);
        }
        if (destDirStr.endsWith(sep)) {
            destDirStr = destDirStr.substring(0, destDirStr.length() - 1);
        }
        FileUtils.getInstance().createDirectory(destDirStr);
        BufferedInputStream in = null;
        BufferedOutputStream out = null;
        byte dataBuff[] = new byte[bufferSize];
        File baseDir = new File(baseDirStr);
        baseDir.mkdirs();
        if (!baseDir.exists()) {
            createDirectory(baseDirStr);
        }
        if ((baseDir.exists()) && (baseDir.isDirectory())) {
            String[] entryList = baseDir.list();
            if (entryList.length > 0) {
                for (int pos = 0; pos < entryList.length; pos++) {
                    String entryName = entryList[pos];
                    String oldPathFileName = baseDirStr + sep + entryName;
                    File entryFile = new File(oldPathFileName);
                    if (entryFile.isFile()) {
                        String newPathFileName = destDirStr + sep + entryName;
                        File newFile = new File(newPathFileName);
                        if (newFile.exists()) {
                            plussQuotaSize -= newFile.length();
                            newFile.delete();
                        }
                        in = new BufferedInputStream(new FileInputStream(oldPathFileName), bufferSize);
                        out = new BufferedOutputStream(new FileOutputStream(newPathFileName), bufferSize);
                        int readLen;
                        while ((readLen = in.read(dataBuff)) > 0) {
                            out.write(dataBuff, 0, readLen);
                            plussQuotaSize += readLen;
                        }
                        out.flush();
                        in.close();
                        out.close();
                    }
                    if (entryFile.isDirectory()) {
                        boolean enableCopyDir = false;
                        if (copyOutputsRtIDsDirs) {
                            enableCopyDir = true;
                        } else {
                            if (entryFile.getParentFile().getName().equals("outputs")) {
                                enableCopyDir = false;
                            } else {
                                enableCopyDir = true;
                            }
                        }
                        if (enableCopyDir) {
                            plussQuotaSize += this.copyDirAllFilesToDirectoryRecursive(baseDirStr + sep + entryName, destDirStr + sep + entryName, copyOutputsRtIDsDirs);
                        }
                    }
                }
            }
        } else {
            throw new Exception("Base dir not exist ! baseDirStr = (" + baseDirStr + ")");
        }
        return plussQuotaSize;
    }
public long copyFile(String baseDirStr, String fileName, String file2FullPath) throws Exception {
        long plussQuotaSize = 0;
        if (!baseDirStr.endsWith(sep)) {
            baseDirStr += sep;
        }
        BufferedInputStream in = null;
        BufferedOutputStream out = null;
        byte dataBuff[] = new byte[bufferSize];
        String file1FullPath = new String(baseDirStr + fileName);
        if (!file1FullPath.equalsIgnoreCase(file2FullPath)) {
            File file1 = new File(file1FullPath);
            if (file1.exists() && (file1.isFile())) {
                File file2 = new File(file2FullPath);
                if (file2.exists()) {
                    plussQuotaSize -= file2.length();
                    file2.delete();
                }
                FileUtils.getInstance().createDirectory(file2.getParent());
                in = new BufferedInputStream(new FileInputStream(file1FullPath), bufferSize);
                out = new BufferedOutputStream(new FileOutputStream(file2FullPath), bufferSize);
                int readLen;
                while ((readLen = in.read(dataBuff)) > 0) {
                    out.write(dataBuff, 0, readLen);
                    plussQuotaSize += readLen;
                }
                out.flush();
                in.close();
                out.close();
            } else {
                throw new Exception("Source file not exist ! file1FullPath = (" + file1FullPath + ")");
            }
        }
        return plussQuotaSize;
    }
public void copyFileToFileWithPaths(String sourcePath, String destinPath) throws Exception {
        BufferedInputStream in = null;
        BufferedOutputStream out = null;
        byte dataBuff[] = new byte[bufferSize];
        File file1 = new File(sourcePath);
        if (file1.exists() && (file1.isFile())) {
            File file2 = new File(destinPath);
            if (file2.exists()) {
                file2.delete();
            }
            FileUtils.getInstance().createDirectory(file2.getParent());
            in = new BufferedInputStream(new FileInputStream(sourcePath), bufferSize);
            out = new BufferedOutputStream(new FileOutputStream(destinPath), bufferSize);
            int readLen;
            while ((readLen = in.read(dataBuff)) > 0) {
                out.write(dataBuff, 0, readLen);
            }
            out.flush();
            in.close();
            out.close();
        } else {
            throw new Exception("Source file not exist ! sourcePath = (" + sourcePath + ")");
        }
    }
private InputStream urlToInputStream(URL url) throws IOException {
        URLConnection conn = url.openConnection();
        conn.setRequestProperty("User-Agent", IE);
        conn.setRequestProperty("Accept-Encoding", "gzip, deflate");
        conn.connect();
        String encoding = conn.getContentEncoding();
        if ((encoding != null) && encoding.equalsIgnoreCase("gzip")) return new GZIPInputStream(conn.getInputStream()); else if ((encoding != null) && encoding.equalsIgnoreCase("deflate")) return new InflaterInputStream(conn.getInputStream(), new Inflater(true)); else return conn.getInputStream();
    }
@Test
    public void testTrim() throws Exception {
        TreeNode ast = TestUtil.readFileInAST("resources/SimpleTestFile.java");
        DecoratorSelection ds = new DecoratorSelection();
        XmlFileSystemRepository rep = new XmlFileSystemRepository();
        XmlToFormatContentConverter converter = new XmlToFormatContentConverter(rep);
        URI url = new File("resources/javaDefaultFormats.xml").toURI();
        InputStream is = url.toURL().openStream();
        converter.convert(is);
        File f = new File("resources/javaDefaultFormats.xml").getAbsoluteFile();
        converter.convert(f);
        String string = new File("resources/query.xml").getAbsolutePath();
        Document qDoc = XmlUtil.loadXmlFromFile(string);
        Query query = new Query(qDoc);
        Format format = XfsrFormatManager.getInstance().getFormats("java", "signature only");
        TokenAutoTrimmer.create("Java", "resources/java.autotrim");
        Document doc = rep.getXmlContentTree(ast, query, format, ds).getOwnerDocument();
        String expected = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?><sourcecode>main(String[])</sourcecode>";
        ByteArrayOutputStream bout = new ByteArrayOutputStream();
        XmlUtil.outputXml(doc, bout);
        String actual = bout.toString();
        assertEquals(expected, actual);
    }
public static void copy(File inputFile, File target) throws IOException {
        if (!inputFile.exists()) return;
        OutputStream output = new FileOutputStream(target);
        InputStream input = new BufferedInputStream(new FileInputStream(inputFile));
        int b;
        while ((b = input.read()) != -1) output.write(b);
        output.close();
        input.close();
    }
public void sendLargePostRequest(final int size) throws IOException {
        String encodedData = URLEncoder.encode("test", WebCastellumParameter.DEFAULT_CHARACTER_ENCODING.getValue()) + "=" + URLEncoder.encode("this is just a mass test", WebCastellumParameter.DEFAULT_CHARACTER_ENCODING.getValue());
        encodedData += "&" + URLEncoder.encode("payload", WebCastellumParameter.DEFAULT_CHARACTER_ENCODING.getValue()) + "=" + createTestdata(size);
        final long start = System.currentTimeMillis();
        final long end;
        HttpURLConnection connection = (HttpURLConnection) this.url.openConnection();
        connection.setDoOutput(true);
        connection.setDoInput(true);
        connection.setUseCaches(false);
        connection.setFollowRedirects(true);
        connection.setInstanceFollowRedirects(true);
        connection.setRequestMethod("POST");
        connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
        connection.setRequestProperty("Content-Length", "" + encodedData.length());
        DataOutputStream output = null;
        BufferedReader reader = null;
        try {
            output = new DataOutputStream(connection.getOutputStream());
            output.writeBytes(encodedData);
            output.flush();
            reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
            String line;
            while ((line = reader.readLine()) != null) {
            }
            end = System.currentTimeMillis();
        } finally {
            if (output != null) {
                try {
                    output.close();
                } catch (IOException ignored) {
                }
            }
            if (reader != null) {
                try {
                    reader.close();
                } catch (IOException ignored) {
                }
            }
        }
        System.out.println("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@");
        System.out.println("Duration: " + (end - start) + " ms");
    }
@Override
    public InitResult init(String name) {
        this.urlString = name;
        URL url;
        URLConnection con;
        try {
            url = new URL(urlString);
            con = url.openConnection();
            int size = con.getContentLength();
            char[] characters = new char[size];
            BufferedReader in = new BufferedReader(new InputStreamReader(url.openStream()));
            in.read(characters);
            in.close();
            return new InitResult(0, size, characters);
        } catch (Exception e) {
            throw new ParserException(e);
        }
    }
@Override
    public void addApplication(Application app) {
        logger.info("Adding a new application " + app.getName() + " by " + app.getOrganisation() + " (" + app.getEmail() + ") ");
        app.setRegtime(new Timestamp(new Date().getTime()));
        try {
            MessageDigest md = MessageDigest.getInstance("SHA-1");
            md.update((app.getName() + app.getEmail() + app.getRegtime()).getBytes());
            byte byteData[] = md.digest();
            StringBuffer sb = new StringBuffer();
            for (int i = 0; i < byteData.length; i++) {
                sb.append(Integer.toString((byteData[i] & 0xff) + 0x100, 16).substring(1));
            }
            app.setAppid(sb.toString());
        } catch (NoSuchAlgorithmException ex) {
            java.util.logging.Logger.getLogger(ApplicationDAOImpl.class.getName()).log(Level.SEVERE, null, ex);
        }
        System.out.println(app.toString());
        SqlParameterSource parameters = new BeanPropertySqlParameterSource(app);
        Number appUid = insertApplication.executeAndReturnKey(parameters);
        app.setId(appUid.longValue());
    }
@Test(dependsOnMethods = { "getSize" })
    public void download() throws IOException {
        FileObject typica = fsManager.resolveFile("s3://" + bucketName + "/jonny.zip");
        File localCache = File.createTempFile("vfs.", ".s3-test");
        FileOutputStream out = new FileOutputStream(localCache);
        IOUtils.copy(typica.getContent().getInputStream(), out);
        Assert.assertEquals(localCache.length(), typica.getContent().getSize());
        localCache.delete();
    }
public String getMD5String(String par1Str) {
        try {
            String s = (new StringBuilder()).append(field_27370_a).append(par1Str).toString();
            MessageDigest messagedigest = MessageDigest.getInstance("MD5");
            messagedigest.update(s.getBytes(), 0, s.length());
            return (new BigInteger(1, messagedigest.digest())).toString(16);
        } catch (NoSuchAlgorithmException nosuchalgorithmexception) {
            throw new RuntimeException(nosuchalgorithmexception);
        }
    }
public static String calculateHash(String data, String algorithm) {
        if (data == null) {
            return null;
        }
        algorithm = (algorithm == null ? INTERNAL : algorithm.toUpperCase());
        if (algorithm.equals(PLAIN)) {
            return data;
        }
        if (algorithm.startsWith("{RSA}")) {
            return encode(data, algorithm.substring(5), "RSA");
        }
        try {
            MessageDigest md = MessageDigest.getInstance(algorithm);
            md.update(data.getBytes("UTF-8"));
            return getHashString(md.digest());
        } catch (UnsupportedEncodingException e) {
            logger.error(e.getMessage());
            return null;
        } catch (NoSuchAlgorithmException nsae) {
            logger.error(nsae.getMessage());
            return null;
        }
    }
public static String md5Encode(String s) {
        try {
            MessageDigest md = MessageDigest.getInstance("MD5");
            md.update(s.getBytes());
            return toHexString(md.digest());
        } catch (NoSuchAlgorithmException e) {
            e.printStackTrace();
            return s;
        }
    }
public static String md5(String value) throws NoSuchAlgorithmException {
        MessageDigest messageDigest = MessageDigest.getInstance("MD5");
        try {
            messageDigest.update(value.getBytes("UTF-8"));
        } catch (UnsupportedEncodingException e) {
            messageDigest.update(value.getBytes());
        }
        byte[] bytes = messageDigest.digest();
        return byteArrayToHexString(bytes);
    }
@Deprecated
    public static Collection<SearchKeyResult> searchKey(String iText, String iKeyServer) throws Exception {
        List<SearchKeyResult> outVec = new ArrayList<SearchKeyResult>();
        String uri = iKeyServer + "/pks/lookup?search=" + URLEncoder.encode(iText, UTF8);
        URL url = new URL(uri);
        BufferedReader input = new BufferedReader(new InputStreamReader(url.openStream()));
        Pattern regex = Pattern.compile("pub.*?<a\\s+href\\s*=\"(.*?)\".*?>\\s*(\\w+)\\s*</a>.*?(\\d+-\\d+-\\d+).*?<a\\s+href\\s*=\".*?\".*?>\\s*(.+?)\\s*</a>", Pattern.CANON_EQ);
        String line;
        while ((line = input.readLine()) != null) {
            Matcher regexMatcher = regex.matcher(line);
            while (regexMatcher.find()) {
                String id = regexMatcher.group(2);
                String downUrl = iKeyServer + regexMatcher.group(1);
                String downDate = regexMatcher.group(3);
                String name = decodeHTML(regexMatcher.group(4));
                outVec.add(new SearchKeyResult(id, name, downDate, downUrl));
            }
        }
        IOUtils.closeQuietly(input);
        return outVec;
    }
public static boolean urlStrIsDir(String urlStr) {
        if (urlStr.endsWith("/")) return true;
        int lastSlash = urlStr.lastIndexOf('/');
        int lastPeriod = urlStr.lastIndexOf('.');
        if (lastPeriod != -1 && (lastSlash == -1 || lastPeriod > lastSlash)) return false;
        String urlStrWithSlash = urlStr + "/";
        try {
            URL url = new URL(urlStrWithSlash);
            InputStream f = url.openStream();
            f.close();
            return true;
        } catch (Exception e) {
            return false;
        }
    }