Skip to content

V
Vietnamese text moderation
Commit 4abc55ff authored by Cao Duc Anh's avatar Cao Duc Anh
Browse files
Options

test ok

parent 7e3872e4
Hide whitespace changes
Inline Side-by-side
Showing with 331 additions and 1 deletion
config.yaml
View file @ 4abc55ff
......@@ -23,7 +23,7 @@ phobert_base:
max_token_length: 256
training:
epoch: 100
epoch: 1200
batch_size: 8
load_data_worker: 2
k_fold: 5
......
lab/test_accuracy.ipynb
View file @ 4abc55ff
......@@ -514,6 +514,170 @@
"acc = total / data_frame.shape[0]\n",
"acc"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Add data"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
"import requests\n",
"import json\n",
"\n",
"def add_data(text, label):\n",
" # URL của API\n",
" url = \"http://10.3.2.100:8008/add-data\"\n",
"\n",
" # Dữ liệu sẽ được gửi dưới dạng JSON\n",
" data = {\n",
" \"paragraph\": text,\n",
" \"label\": label\n",
" }\n",
"\n",
" # Gửi yêu cầu POST tới API\n",
" response = requests.post(url, headers={\"Content-Type\": \"application/json\"}, data=json.dumps(data))\n",
"\n",
" # In ra kết quả trả về từ API\n",
" if response.status_code == 200:\n",
" print(\"Response from API:\", response.json())\n",
" else:\n",
" print(\"Failed to call API:\", response.status_code, response.text)"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"text = \"csv\"\n",
"label = \"khac\""
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Response from API: {'paragraph': 'csv', 'label': 'khac'}\n"
]
}
],
"source": [
"add_data(text, label)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Training"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [],
"source": [
"import requests\n",
"import json\n",
"\n",
"def start_training(pretrain):\n",
" # URL của API\n",
" url = \"http://10.3.2.100:8000/start-training\"\n",
"\n",
" # Dữ liệu sẽ được gửi dưới dạng JSON\n",
" data = {\n",
" \"pretrain\": pretrain,\n",
" }\n",
"\n",
" # Gửi yêu cầu POST tới API\n",
" response = requests.post(url, headers={\"Content-Type\": \"application/json\"}, data=json.dumps(data))\n",
"\n",
" # In ra kết quả trả về từ API\n",
" if response.status_code == 200:\n",
" print(\"Response from API:\", response.json())\n",
" else:\n",
" print(\"Failed to call API:\", response.status_code, response.text)"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Response from API: {'status': 'Training started', 'message': 'Tracking and visualizing metrics on TensorBoard UI: http://server_ip:6006/'}\n"
]
}
],
"source": [
"start_training(\"latest\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Stop training"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {},
"outputs": [],
"source": [
"import requests\n",
"import json\n",
"\n",
"def stop_training():\n",
" # URL của API\n",
" url = \"http://10.3.2.100:8000/stop-training\"\n",
"\n",
" # Dữ liệu sẽ được gửi dưới dạng JSON\n",
"\n",
" # Gửi yêu cầu POST tới API\n",
" response = requests.post(url)\n",
"\n",
" # In ra kết quả trả về từ API\n",
" if response.status_code == 200:\n",
" print(\"Response from API:\", response.json())\n",
" else:\n",
" print(\"Failed to call API:\", response.status_code, response.text)"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Response from API: {'status': 'Training stopped', 'message': 'VRam GPU are released'}\n"
]
}
],
"source": [
"stop_training()"
]
}
],
"metadata": {
......
nginx/log/access.log
View file @ 4abc55ff
......@@ -97,3 +97,4 @@
10.3.2.17 - - [12/Sep/2024:02:52:17 +0000] "POST /text-classify HTTP/1.1" 200 67 "-" "PostmanRuntime/7.42.0" "-"
10.3.2.100 - - [12/Sep/2024:03:01:49 +0000] "POST /text-classify HTTP/1.1" 200 67 "-" "curl/7.81.0" "-"
10.3.2.100 - - [12/Sep/2024:03:02:00 +0000] "POST /text-classify HTTP/1.1" 200 67 "-" "curl/7.81.0" "-"
10.3.2.100 - - [16/Sep/2024:06:31:31 +0000] "POST /text-classify HTTP/1.1" 200 2292 "-" "curl/7.81.0" "-"
nginx/log/error.log
View file @ 4abc55ff
......@@ -8840,3 +8840,165 @@
2024/09/16 03:08:47 [notice] 1#1: signal 17 (SIGCHLD) received from 24
2024/09/16 03:08:47 [notice] 1#1: worker process 24 exited with code 0
2024/09/16 03:08:47 [notice] 1#1: exit
2024/09/16 03:18:55 [notice] 1#1: using the "epoll" event method
2024/09/16 03:18:55 [notice] 1#1: nginx/1.25.0
2024/09/16 03:18:55 [notice] 1#1: built by gcc 10.2.1 20210110 (Debian 10.2.1-6)
2024/09/16 03:18:55 [notice] 1#1: OS: Linux 6.5.0-45-generic
2024/09/16 03:18:55 [notice] 1#1: getrlimit(RLIMIT_NOFILE): 1048576:1048576
2024/09/16 03:18:55 [notice] 1#1: start worker processes
2024/09/16 03:18:55 [notice] 1#1: start worker process 20
2024/09/16 03:18:55 [notice] 1#1: start worker process 21
2024/09/16 03:18:55 [notice] 1#1: start worker process 22
2024/09/16 03:18:55 [notice] 1#1: start worker process 23
2024/09/16 03:18:55 [notice] 1#1: start worker process 24
2024/09/16 03:18:55 [notice] 1#1: start worker process 25
2024/09/16 03:18:55 [notice] 1#1: start worker process 26
2024/09/16 03:18:55 [notice] 1#1: start worker process 27
2024/09/16 03:18:55 [notice] 1#1: start worker process 28
2024/09/16 03:18:55 [notice] 1#1: start worker process 29
2024/09/16 03:18:55 [notice] 1#1: start worker process 30
2024/09/16 03:18:55 [notice] 1#1: start worker process 31
2024/09/16 07:01:32 [notice] 1#1: signal 3 (SIGQUIT) received, shutting down
2024/09/16 07:01:32 [notice] 20#20: gracefully shutting down
2024/09/16 07:01:32 [notice] 21#21: gracefully shutting down
2024/09/16 07:01:32 [notice] 22#22: gracefully shutting down
2024/09/16 07:01:32 [notice] 23#23: gracefully shutting down
2024/09/16 07:01:32 [notice] 24#24: gracefully shutting down
2024/09/16 07:01:32 [notice] 25#25: gracefully shutting down
2024/09/16 07:01:32 [notice] 26#26: gracefully shutting down
2024/09/16 07:01:32 [notice] 20#20: exiting
2024/09/16 07:01:32 [notice] 27#27: gracefully shutting down
2024/09/16 07:01:32 [notice] 21#21: exiting
2024/09/16 07:01:32 [notice] 22#22: exiting
2024/09/16 07:01:32 [notice] 23#23: exiting
2024/09/16 07:01:32 [notice] 24#24: exiting
2024/09/16 07:01:32 [notice] 25#25: exiting
2024/09/16 07:01:32 [notice] 26#26: exiting
2024/09/16 07:01:32 [notice] 27#27: exiting
2024/09/16 07:01:32 [notice] 31#31: gracefully shutting down
2024/09/16 07:01:32 [notice] 20#20: exit
2024/09/16 07:01:32 [notice] 28#28: gracefully shutting down
2024/09/16 07:01:32 [notice] 23#23: exit
2024/09/16 07:01:32 [notice] 22#22: exit
2024/09/16 07:01:32 [notice] 24#24: exit
2024/09/16 07:01:32 [notice] 21#21: exit
2024/09/16 07:01:32 [notice] 25#25: exit
2024/09/16 07:01:32 [notice] 27#27: exit
2024/09/16 07:01:32 [notice] 26#26: exit
2024/09/16 07:01:32 [notice] 28#28: exiting
2024/09/16 07:01:32 [notice] 31#31: exiting
2024/09/16 07:01:32 [notice] 28#28: exit
2024/09/16 07:01:32 [notice] 31#31: exit
2024/09/16 07:01:32 [notice] 29#29: gracefully shutting down
2024/09/16 07:01:32 [notice] 29#29: exiting
2024/09/16 07:01:32 [notice] 29#29: exit
2024/09/16 07:01:32 [notice] 30#30: gracefully shutting down
2024/09/16 07:01:32 [notice] 30#30: exiting
2024/09/16 07:01:32 [notice] 30#30: exit
2024/09/16 07:01:32 [notice] 1#1: signal 17 (SIGCHLD) received from 26
2024/09/16 07:01:32 [notice] 1#1: worker process 26 exited with code 0
2024/09/16 07:01:32 [notice] 1#1: signal 29 (SIGIO) received
2024/09/16 07:01:32 [notice] 1#1: signal 17 (SIGCHLD) received from 29
2024/09/16 07:01:32 [notice] 1#1: worker process 29 exited with code 0
2024/09/16 07:01:32 [notice] 1#1: signal 29 (SIGIO) received
2024/09/16 07:01:32 [notice] 1#1: signal 17 (SIGCHLD) received from 31
2024/09/16 07:01:32 [notice] 1#1: worker process 31 exited with code 0
2024/09/16 07:01:32 [notice] 1#1: worker process 21 exited with code 0
2024/09/16 07:01:32 [notice] 1#1: worker process 22 exited with code 0
2024/09/16 07:01:32 [notice] 1#1: worker process 24 exited with code 0
2024/09/16 07:01:32 [notice] 1#1: worker process 25 exited with code 0
2024/09/16 07:01:32 [notice] 1#1: worker process 27 exited with code 0
2024/09/16 07:01:32 [notice] 1#1: signal 29 (SIGIO) received
2024/09/16 07:01:32 [notice] 1#1: signal 17 (SIGCHLD) received from 22
2024/09/16 07:01:32 [notice] 1#1: signal 17 (SIGCHLD) received from 28
2024/09/16 07:01:32 [notice] 1#1: worker process 28 exited with code 0
2024/09/16 07:01:32 [notice] 1#1: signal 29 (SIGIO) received
2024/09/16 07:01:32 [notice] 1#1: signal 17 (SIGCHLD) received from 30
2024/09/16 07:01:32 [notice] 1#1: worker process 30 exited with code 0
2024/09/16 07:01:32 [notice] 1#1: worker process 20 exited with code 0
2024/09/16 07:01:32 [notice] 1#1: worker process 23 exited with code 0
2024/09/16 07:01:32 [notice] 1#1: exit
2024/09/16 07:06:13 [notice] 1#1: using the "epoll" event method
2024/09/16 07:06:13 [notice] 1#1: nginx/1.25.0
2024/09/16 07:06:13 [notice] 1#1: built by gcc 10.2.1 20210110 (Debian 10.2.1-6)
2024/09/16 07:06:13 [notice] 1#1: OS: Linux 6.5.0-45-generic
2024/09/16 07:06:13 [notice] 1#1: getrlimit(RLIMIT_NOFILE): 1048576:1048576
2024/09/16 07:06:13 [notice] 1#1: start worker processes
2024/09/16 07:06:13 [notice] 1#1: start worker process 20
2024/09/16 07:06:13 [notice] 1#1: start worker process 21
2024/09/16 07:06:13 [notice] 1#1: start worker process 22
2024/09/16 07:06:13 [notice] 1#1: start worker process 23
2024/09/16 07:06:13 [notice] 1#1: start worker process 24
2024/09/16 07:06:13 [notice] 1#1: start worker process 25
2024/09/16 07:06:13 [notice] 1#1: start worker process 26
2024/09/16 07:06:13 [notice] 1#1: start worker process 27
2024/09/16 07:06:13 [notice] 1#1: start worker process 28
2024/09/16 07:06:13 [notice] 1#1: start worker process 29
2024/09/16 07:06:13 [notice] 1#1: start worker process 30
2024/09/16 07:06:13 [notice] 1#1: start worker process 31
2024/09/16 07:57:17 [notice] 1#1: signal 3 (SIGQUIT) received, shutting down
2024/09/16 07:57:17 [notice] 20#20: gracefully shutting down
2024/09/16 07:57:17 [notice] 21#21: gracefully shutting down
2024/09/16 07:57:17 [notice] 22#22: gracefully shutting down
2024/09/16 07:57:17 [notice] 23#23: gracefully shutting down
2024/09/16 07:57:17 [notice] 27#27: gracefully shutting down
2024/09/16 07:57:17 [notice] 24#24: gracefully shutting down
2024/09/16 07:57:17 [notice] 30#30: gracefully shutting down
2024/09/16 07:57:17 [notice] 26#26: gracefully shutting down
2024/09/16 07:57:17 [notice] 20#20: exiting
2024/09/16 07:57:17 [notice] 21#21: exiting
2024/09/16 07:57:17 [notice] 22#22: exiting
2024/09/16 07:57:17 [notice] 23#23: exiting
2024/09/16 07:57:17 [notice] 27#27: exiting
2024/09/16 07:57:17 [notice] 24#24: exiting
2024/09/16 07:57:17 [notice] 30#30: exiting
2024/09/16 07:57:17 [notice] 26#26: exiting
2024/09/16 07:57:17 [notice] 21#21: exit
2024/09/16 07:57:17 [notice] 20#20: exit
2024/09/16 07:57:17 [notice] 22#22: exit
2024/09/16 07:57:17 [notice] 23#23: exit
2024/09/16 07:57:17 [notice] 30#30: exit
2024/09/16 07:57:17 [notice] 27#27: exit
2024/09/16 07:57:17 [notice] 24#24: exit
2024/09/16 07:57:17 [notice] 26#26: exit
2024/09/16 07:57:17 [notice] 25#25: gracefully shutting down
2024/09/16 07:57:17 [notice] 25#25: exiting
2024/09/16 07:57:17 [notice] 25#25: exit
2024/09/16 07:57:17 [notice] 28#28: gracefully shutting down
2024/09/16 07:57:17 [notice] 28#28: exiting
2024/09/16 07:57:17 [notice] 29#29: gracefully shutting down
2024/09/16 07:57:17 [notice] 31#31: gracefully shutting down
2024/09/16 07:57:17 [notice] 29#29: exiting
2024/09/16 07:57:17 [notice] 28#28: exit
2024/09/16 07:57:17 [notice] 31#31: exiting
2024/09/16 07:57:17 [notice] 29#29: exit
2024/09/16 07:57:17 [notice] 31#31: exit
2024/09/16 07:57:17 [notice] 1#1: signal 17 (SIGCHLD) received from 25
2024/09/16 07:57:17 [notice] 1#1: worker process 25 exited with code 0
2024/09/16 07:57:17 [notice] 1#1: worker process 29 exited with code 0
2024/09/16 07:57:17 [notice] 1#1: signal 29 (SIGIO) received
2024/09/16 07:57:17 [notice] 1#1: signal 17 (SIGCHLD) received from 31
2024/09/16 07:57:17 [notice] 1#1: worker process 31 exited with code 0
2024/09/16 07:57:17 [notice] 1#1: signal 29 (SIGIO) received
2024/09/16 07:57:17 [notice] 1#1: signal 17 (SIGCHLD) received from 21
2024/09/16 07:57:17 [notice] 1#1: worker process 21 exited with code 0
2024/09/16 07:57:17 [notice] 1#1: signal 29 (SIGIO) received
2024/09/16 07:57:17 [notice] 1#1: signal 17 (SIGCHLD) received from 20
2024/09/16 07:57:17 [notice] 1#1: worker process 20 exited with code 0
2024/09/16 07:57:17 [notice] 1#1: worker process 23 exited with code 0
2024/09/16 07:57:17 [notice] 1#1: worker process 28 exited with code 0
2024/09/16 07:57:17 [notice] 1#1: signal 29 (SIGIO) received
2024/09/16 07:57:17 [notice] 1#1: signal 17 (SIGCHLD) received from 23
2024/09/16 07:57:17 [notice] 1#1: signal 17 (SIGCHLD) received from 27
2024/09/16 07:57:17 [notice] 1#1: worker process 27 exited with code 0
2024/09/16 07:57:17 [notice] 1#1: signal 29 (SIGIO) received
2024/09/16 07:57:17 [notice] 1#1: signal 17 (SIGCHLD) received from 30
2024/09/16 07:57:17 [notice] 1#1: worker process 22 exited with code 0
2024/09/16 07:57:17 [notice] 1#1: worker process 30 exited with code 0
2024/09/16 07:57:17 [notice] 1#1: signal 29 (SIGIO) received
2024/09/16 07:57:17 [notice] 1#1: signal 17 (SIGCHLD) received from 24
2024/09/16 07:57:17 [notice] 1#1: worker process 24 exited with code 0
2024/09/16 07:57:17 [notice] 1#1: signal 29 (SIGIO) received
2024/09/16 07:57:17 [notice] 1#1: signal 17 (SIGCHLD) received from 26
2024/09/16 07:57:17 [notice] 1#1: worker process 26 exited with code 0
2024/09/16 07:57:17 [notice] 1#1: exit
server_manage_data/requirements.txt
View file @ 4abc55ff
fastapi
uvicorn
sqlalchemy
asyncpg
psycopg2-binary
minio==7.2.7
pandas==2.2.1
PyYAML
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment