llm_icd_coding_data/checkdata.py at main · macqueen09/llm_icd_coding_data · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
import pandas as pd
import json
import csv
from tqdm import tqdm

from util_db import DBApi

def func():
    path = '/supercloud/llm-data/icd10paper/4_mimic_iv/csv_note/'
    filename = 'discharge.csv' # (331793, 8)

    nrows=None
    nrows=300
    df = pd.read_csv(path+filename,nrows=nrows)
    print(df)
    print(df.shape)

    hadmid = df["hadm_id"]
    print(hadmid)
    print(hadmid.shape)
    h = hadmid.to_list()
    print(len(h))
    print(len(set(h)))

    return set(h)


def get_id():
    path = '/supercloud/llm-data/icd10paper/4_mimic_iv/mimic-iv-3.1/csv_hosp/'
    filename = 'diagnoses_icd.csv' # (6364488, 5)
    #filename = 'd_icd_diagnoses.csv' # (112107, 3)

    nrows=None
    nrows=300
    df = pd.read_csv(path+filename,nrows=nrows)
    print(df)
    print(df.shape)
    a = df[df["icd_version"] == 10]
    #print(a)
    h_list = a["hadm_id"]
    print(len(set(h_list)))

    icd10id = set(h_list)
    return icd10id


#textid = func()
#icd10id = get_id()
#
#res = textid & icd10id
#print("--")
#print(len(res))


def get_has_id():
    path = '/supercloud/llm-data/icd10paper/4_mimic_iv/mimic-iv-3.1/csv_hosp/'
    #filename = 'diagnoses_icd.csv' # (6364488, 5)
    filename = 'd_icd_diagnoses.csv' # (112107, 3)

    nrows=None
    nrows=100
    df = pd.read_csv(path+filename,nrows=nrows)
    print(df)
    print(df.shape)

get_has_id()