Use PHP to batch-import millions of TXT files into a database.

Currently, the MySQL database is an excellent choice for storing data; therefore, the code is as follows:

<?php
// 调用函数
// moveFilesByExtension('./回收站', './句子采集', '.txt');die;

// 数据库连接信息
$servername = "localhost";
$username = "";
$password = "";
$dbname = "";

// 创建数据库连接
$conn = new mysqli($servername, $username, $password, $dbname);
if ($conn->connect_error) {
    die("数据库连接失败: " . $conn->connect_error);
}

// 设置每次读取的文件数量
$fileLimit = 100;

// 打开文件夹
$folderPath = './句子采集/';  // 替换为实际的文件夹路径
if ($dh = opendir($folderPath)) {
    $fileCount = 0;  // 计算已读取的文件数量
    while (($file = readdir($dh)) !== false) {
        if ($file != '.' && $file != '..' && pathinfo($file, PATHINFO_EXTENSION) == 'txt') {
            $filePath = $folderPath . $file;
            $content = file_get_contents($filePath);
            $title = basename($filePath);
           // 将文件内容插入数据库表
            $sql = "INSERT INTO caiji_juzi (title, content) VALUES ('" . $conn->real_escape_string($title) . "', '" . $conn->real_escape_string($content) . "')";   // 根据你的表结构修改表名和字段名
            if ($conn->query($sql) === TRUE) {
                if (rename($filePath, str_replace('句子采集', '回收站', $filePath))) {
                    echo "文件 " . $file . " 插入成功并移动到回收站<br>";
                    echo '<meta http-equiv="refresh" content="1">';
                } else {
                    echo "文件 " . $file . " 插入成功但移动失败<br>";
                }
            } else {
                echo "文件 " . $file . " 插入失败: " . $conn->error . "<br>";
            }

            $fileCount++;

            // 如果已读取的文件数量达到设定的限制，停止读取
            if ($fileCount >= $fileLimit) {
                break;
            }
        }
    }
    closedir($dh);
}

// 关闭数据库连接
$conn->close();

function moveFilesByExtension($sourceDir, $targetDir, $fileExtension) {
    if (!is_dir($sourceDir) || !is_dir($targetDir)) {
        echo '源文件夹或目标文件夹不存在';
    } else {
        if ($dh = opendir($sourceDir)) {
            while (($file = readdir($dh)) !== false) {
                if (substr($file, -strlen($fileExtension)) === $fileExtension) {
                    rename("$sourceDir/$file", "$targetDir/$file");
                }
            }
            closedir($dh);
        }
    }
}