I am trying to create a program that downloads a JSON file and I'm trying to convert it to sqlite.

Why?:

  • I believe the training data on Deepdanbooru is weak compaired to Rule34 or Gelbooru. I have tried compiling the C# for linux, but I decided it was too hard.

I am a mainly a Web developer who uses basic JS/HTML/CSS. These don't help as I need it to be in C. (Not ++/#). I have tried using AI to assist, but no matter what language I use (Python, Ruby, Java, C), It will fail.

Here is the closest I gotten ( 〔〕 --> ><:

#include 〔stdio.h〕
#include 〔stdlib.h〕
#include 〔string.h〕
#include 〔curl/curl.h〕
#include &quot;cJSON.h&quot;
#include &quot;sqlite3.h&quot;

#define URL &quot;https://danbooru.donmai.us/posts.json?page=1&amp;limit=1000&amp;tags=duck&amp;json=1&quot;

#define DB_NAME &quot;data.db&quot;

static int callback(void *NotUsed, int argc, char **argv, char **azColName) {
    int i;
    for (i = 0; i &lt; argc; i++) {
        printf(&quot;%s = %s\n&quot;, azColName[i], argv[i] ? argv[i] : &quot;NULL&quot;);
    }
    printf(&quot;\n&quot;);
    return 0;
}

int main() {
    CURL *curl;
    CURLcode res;
    FILE *fp;
    char *url = URL;
    char outfilename[FILENAME_MAX] = &quot;data.json&quot;;
    curl = curl_easy_init();
    if (curl) {
        fp = fopen(outfilename, &quot;wb&quot;);
        curl_easy_setopt(curl, CURLOPT_URL, url);
        curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
        curl_easy_setopt(curl, CURLOPT_USERAGENT, &quot;Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0)&quot;);
        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, NULL);
        curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
        res = curl_easy_perform(curl);
        curl_easy_cleanup(curl);
        fclose(fp);
    }
    sqlite3 *db;
    char *err_msg = 0;
    int rc = sqlite3_open(DB_NAME, &amp;db);
    if (rc != SQLITE_OK) {
        fprintf(stderr, &quot;Cannot open database: %s\n&quot;, sqlite3_errmsg(db));
        sqlite3_close(db);
        return 1;
    }
    char *sql = &quot;CREATE TABLE data (id INT PRIMARY KEY NOT NULL, md5 TEXT NOT NULL, tag_string TEXT NOT NULL, tag_count_general INT NOT NULL);&quot;;
    if (rc != SQLITE_OK) {
        fprintf(stderr, &quot;SQL error: %s\n&quot;, err_msg);
        sqlite3_free(err_msg);
        sqlite3_close(db);
        return 1;
    }
    cJSON *json;
    cJSON *item;
    cJSON *id;
    cJSON *md5;
    cJSON *tag_string;
    cJSON *tag_count_general;
    char buffer[1024];
    fp = fopen(&quot;data.json&quot;, &quot;r&quot;);
    fread(buffer, 1024, 1, fp);
    fclose(fp);
    json = cJSON_Parse(buffer);
    if (!json) {
        printf(&quot;Error before: [%s]\n&quot;, cJSON_GetErrorPtr());
        return 1;
    }
    cJSON_ArrayForEach(item, json) {
        id = cJSON_GetObjectItem(item, &quot;id&quot;);
        md5 = cJSON_GetObjectItem(item, &quot;md5&quot;);
        tag_string = cJSON_GetObjectItem(item, &quot;tag_string&quot;);
        tag_count_general = cJSON_GetObjectItem(item, &quot;tag_count_general&quot;);
        char insert_query[1024];
        sprintf(insert_query,
                &quot;INSERT INTO data (id, md5, tag_string, tag_count_general) VALUES (%d,&#x27;%s&#x27;,&#x27;%s&#x27;,%d);&quot;,
                id->valueint,
                md5->valuestring,
                tag_string->valuestring,
                tag_count_general->valueint
               );
        rc = sqlite3_exec(db, insert_query, callback, 0, &amp;err_msg);
        if (rc != SQLITE_OK) {
            fprintf(stderr, &quot;SQL error: %s\n&quot;, err_msg);
            sqlite3_free(err_msg);
            sqlite3_close(db);
            return 1;
        }
    }
}

Compile: gcc cJSON.c file.c -lcurl -lsqlite3

Error: Error before: [tag_count_co]

  • farcaller@fstab.sh
    link
    fedilink
    arrow-up
    3
    arrow-down
    1
    ·
    11 months ago

    Streaming JSON parsers are a thing, e.g. pdjson for C. It's, of course, a different approach and it's generally slightly trickier to work with those, but that's what you would use of you have unbound document size and you can process it in chunks.