diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f65d16d --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +#.idea +.idea/* \ No newline at end of file diff --git a/.idea/NotebookServer.iml b/.idea/NotebookServer.iml index 6711606..0a230a0 100644 --- a/.idea/NotebookServer.iml +++ b/.idea/NotebookServer.iml @@ -2,7 +2,7 @@ - + diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 0000000..0820a83 --- /dev/null +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,14 @@ + + + + \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml index 6c712d2..c00c3c0 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -1,9 +1,12 @@ + + + - - + + - - - + + + + + - - + + + + + - @@ -103,50 +166,76 @@ - + - - +<<<<<<< HEAD + +======= + +>>>>>>> class + + + + + - + - - + + - + + + + - + - - +<<<<<<< HEAD + + + + + +======= + + - + +>>>>>>> class - + - - - - - +<<<<<<< HEAD + + +======= + + +>>>>>>> class + + + + + @@ -158,36 +247,36 @@ - createProject - getAllFiles - ls -l|grep - subpro - subprocess - getFileNumber - size - syso - ill - getFilesInfoOfPath - sysout - SubProcessCmd - shell.S - PRINT - print + MKFILE + sleep + json.dumps + md5 + ws_server + movef + resetDatasets + copy + pjIdsInClass + pathUser + copyClassDatasets + copyClassProject + dir_pub_class + dataset + copyClassData + mode + projects + threading + bindDataWithProject + base_id + config/ws + Id + coursewareId getDirSize - utf - sysout. - .log( - cmd is none - start vm failed! cause: - rm -rf - resp_err_version_invlid - getDirN - incorrect version number! - ns_doname - deleteFiles - getDirNumber - mPort - ns_port_http + delete + resetClassProject + bind + pathPj + createnew + getNotebookName @@ -207,7 +296,6 @@ @@ -238,10 +340,17 @@ true DEFINITION_ORDER - - - + - + - + - + - + - - + + + - - + - - + + @@ -459,9 +616,28 @@ + +<<<<<<< HEAD + @@ -472,31 +648,46 @@ - + - + + + + + - - - - + + +<<<<<<< HEAD + + +======= + + +>>>>>>> class - + - - - - - - - + + + + + + + + + + + + + @@ -531,203 +722,68 @@ + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + - + - - + + - + - - - - - - - - - - + - - - - - - + + - + - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - - + + - - - - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - + - - + + @@ -738,162 +794,249 @@ - + + - - + + - - - - - - + + + + + + + + + + + + - - + + - - + + - + - - + + - + + + - - + + - + + + + - - + + - + + + + + + - - + + - + + - - - + - + - - + + - - - - + - - - - - + + - +<<<<<<< HEAD + - - + + +======= + + + + + - - + + - - - - - - + +>>>>>>> class - - + + - + + + + - +<<<<<<< HEAD + - - + + - + + - - - - - - - - - +======= + + - + - - + + +>>>>>>> class - + - - +<<<<<<< HEAD + + - - - - - - - - - + + +======= + + - + +>>>>>>> class + + + + + + + + + + + + + No facets are configured + + + + + + + + Python 3.7 interpreter library + + + + + + + + 1.8 (1) + + + + + + + + NotebookServer + + + + + + + + + + + + + + \ No newline at end of file diff --git a/bin/autodeeplearning.sh b/bin/autodeeplearning.sh index fbb1e1d..8bb17dc 100644 --- a/bin/autodeeplearning.sh +++ b/bin/autodeeplearning.sh @@ -15,10 +15,10 @@ pm2 start processes.json # auto start notebook server ## release server -python3 /notebook/NotebookServer/project/main/main.py +python3 /notebook/NotebookServer/project/main/startup.py ## dev server -#python3 ~/notebook/NotebookServer/project/main/main.py +#python3 /notebook/NotebookServer/project/main/startup.py ## jupyter notebook jupyter notebook --allow-root --ip=0.0.0.0 --no-browser diff --git a/docs/install.md b/docs/install.md new file mode 100644 index 0000000..8505a9f --- /dev/null +++ b/docs/install.md @@ -0,0 +1,5 @@ +# Ubuntu Server 1604 set system encoding +# +# cmd : $dpkg-reconfigure locales +# choose: zh_CN.UTF-8 +# done \ No newline at end of file diff --git a/logs/NotebookServer.log b/logs/NotebookServer.log deleted file mode 100644 index e69de29..0000000 diff --git a/project/main/base/__pycache__/__init__.cpython-37.pyc b/project/main/base/__pycache__/__init__.cpython-37.pyc new file mode 100644 index 0000000..1c4e367 Binary files /dev/null and b/project/main/base/__pycache__/__init__.cpython-37.pyc differ diff --git a/project/main/base/__pycache__/sysout.cpython-36.pyc b/project/main/base/__pycache__/sysout.cpython-36.pyc index 439746b..75c6edf 100644 Binary files a/project/main/base/__pycache__/sysout.cpython-36.pyc and b/project/main/base/__pycache__/sysout.cpython-36.pyc differ diff --git a/project/main/base/__pycache__/sysout.cpython-37.pyc b/project/main/base/__pycache__/sysout.cpython-37.pyc new file mode 100644 index 0000000..62ddf49 Binary files /dev/null and b/project/main/base/__pycache__/sysout.cpython-37.pyc differ diff --git a/project/main/base/sysout.py b/project/main/base/sysout.py index b4db845..06eede4 100644 --- a/project/main/base/sysout.py +++ b/project/main/base/sysout.py @@ -1,15 +1,16 @@ # coding:utf-8 import logging -import os, sys +import os, sys, time + # from config import config # path = os.getcwd().split("/project")[0]+'/logs' -path = sys.argv[0].split("/project")[0]+'/logs' +path = sys.argv[0].split("/project")[0] + '/logs' logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s', datefmt='%Y/%m/%d %H:%M:%S', - filename= path + '/NotebookServer.log', + filename=path + '/NotebookServer-' + time.strftime("%Y.%m.%d-%H:%M:%S", time.localtime()) + '.log', # filename= config.logpath, filemode='w') logger = logging.getLogger("NBLogger") @@ -24,7 +25,7 @@ def log(tag, msg): print(log) logging.log(log) except Exception as e: - print('Exception:'+str(e)) + print('Exception:' + str(e)) logging.error(str(e)) @@ -35,7 +36,7 @@ def err(tag, err): print(log) logging.debug(log) except Exception as e: - print('Exception:'+str(e)) + print('Exception:' + str(e)) logging.error(str(e)) @@ -46,5 +47,5 @@ def info(tag, msg): print(log) logging.info(log) except Exception as e: - print('Exception:'+str(e)) + print('Exception:' + str(e)) logging.error(str(e)) diff --git a/project/main/config/__pycache__/__init__.cpython-37.pyc b/project/main/config/__pycache__/__init__.cpython-37.pyc new file mode 100644 index 0000000..7f2640d Binary files /dev/null and b/project/main/config/__pycache__/__init__.cpython-37.pyc differ diff --git a/project/main/config/__pycache__/config.cpython-36.pyc b/project/main/config/__pycache__/config.cpython-36.pyc index 4fac359..43b43fa 100644 Binary files a/project/main/config/__pycache__/config.cpython-36.pyc and b/project/main/config/__pycache__/config.cpython-36.pyc differ diff --git a/project/main/config/__pycache__/config.cpython-37.pyc b/project/main/config/__pycache__/config.cpython-37.pyc new file mode 100644 index 0000000..cba973c Binary files /dev/null and b/project/main/config/__pycache__/config.cpython-37.pyc differ diff --git a/project/main/config/__pycache__/string.cpython-37.pyc b/project/main/config/__pycache__/string.cpython-37.pyc new file mode 100644 index 0000000..db8b7b4 Binary files /dev/null and b/project/main/config/__pycache__/string.cpython-37.pyc differ diff --git a/project/main/config/config.py b/project/main/config/config.py index aad136d..f3456b8 100644 --- a/project/main/config/config.py +++ b/project/main/config/config.py @@ -7,9 +7,9 @@ 'app_name': "极算云notebook文件管理分发系统", 'version': "v1.0.1", 'author': "JayYin", - 'mode': 'DEV' + # 'mode': 'DEV' # 'mode': 'RELEASE' - # 'mode': 'LOCALHOST' + 'mode': 'LOCALHOST' } # server @@ -42,12 +42,22 @@ dir_home_user = '' file_system_readme = dir_home + '/base/readme' +#/notebook/storage/base/class/classIds/projectIds +#/notebook/storage/base/datasets/ +dir_pub_class = dir_home + '/base/class' +dir_pub_dsets = dir_home + '/base/datasets' +# +# + + + # ----------- RELEASE ------------ ns_config_release = { - 'ns_host': '172.16.59.99', # _release server cloudyotech.com', + 'ns_host': '60.12.136.58', # _release server cloudyotech.com', + 'ns_host_pub': '120.26.57.100', 'ns_port_http': 8100, 'ns_port': 8888, - 'ns_doname ': 'https://g.cloudyotech.com/notebook', + 'ns_doname': 'https://f.cloudyotech.com/notebook', # system home dir 'dir_home': '/notebook/storage', 'dir_home_user': '', @@ -57,10 +67,13 @@ # ----------- DEV ------------ ns_config_dev = { - 'ns_host': '172.16.3.254', + # 'ns_host': '60.12.136.58', #18.174 ubuntu + 'ns_host': '18.18.45.175', #18.174 ubuntu + # 'ns_host': '172.16.3.254', #aliyun dev server + 'ns_host_pub': '120.26.48.110', 'ns_port_http': 8100, 'ns_port': 8888, - 'ns_doname': 'https://dev.dongxicc.cn/notebook', + 'ns_doname': 'https://file.dongxicc.cn/notebook', # system home dir 'dir_home': '/notebook/storage', 'dir_home_user': '', @@ -71,6 +84,7 @@ # ----------- DEV2 for 18.18.18.174 ------------ ns_config_dev2 = { 'ns_host': '18.18.18.174', + 'ns_host_pub': '120.26.48.110', 'ns_port_http': 8100, 'ns_port': 8888, 'ns_doname': 'https://18.18.18.174:8888', @@ -84,22 +98,25 @@ # ----------- localohst ------------ ns_config_local = { 'ns_host': '127.0.0.1', + 'ns_host_pub': '120.26.48.110', 'ns_port_http': 8100, 'ns_port': 8888, - 'ns_doname': 'http://127.0.0.1:8888', + 'ns_doname': 'https://127.0.0.1:8888', # system home dir - 'dir_home': '/Users/jerryyin/workspace/notebook/storage-dev', + 'dir_home': '/Users/jerryyin/workspace/notebook/storage', 'dir_home_user': '', - 'file_system_readme': '/Users/jerryyin/workspace/notebook/storage-dev/base/readme' + 'file_system_readme': '/Users/jerryyin/workspace/notebook/storage/base/readme' + # 'dir_pub_class': '/Users/jerryyin/workspace/notebook/storage/base/class', + # 'dir_pub_dsets': '/Users/jerryyin/workspace/notebook/storage/base/datasets' } def initConfig(): sysout.info(TAG, 'init system config of model [%s] ..' % system['mode']) - global vms_host, ns_host, ns_port, ns_port_http, ns_doname, dir_home, dir_home_user, file_system_readme + global vms_host,ns_host_pub, ns_host, ns_port, ns_port_http, ns_doname, dir_home, dir_home_user, file_system_readme, dir_pub_class, dir_pub_dsets if system['mode'] == 'DEV': vms_host = vms_host_dev - + ns_host_pub = ns_config_dev['ns_host_pub'] ns_host = ns_config_dev['ns_host'] ns_port_http = ns_config_dev['ns_port_http'] ns_port = ns_config_dev['ns_port'] @@ -109,7 +126,7 @@ def initConfig(): file_system_readme = ns_config_dev['file_system_readme'] elif system['mode'] == 'RELEASE': vms_host = vms_host_release - + ns_host_pub = ns_config_release['ns_host_pub'] ns_host = ns_config_release['ns_host'] ns_port_http = ns_config_release['ns_port_http'] ns_port = ns_config_release['ns_port'] @@ -119,7 +136,7 @@ def initConfig(): file_system_readme = ns_config_release['file_system_readme'] elif system['mode'] == 'LOCALHOST': vms_host = vms_host_dev - + ns_host_pub = ns_config_local['ns_host_pub'] ns_host = ns_config_local['ns_host'] ns_port_http = ns_config_local['ns_port_http'] ns_port = ns_config_local['ns_port'] @@ -127,6 +144,8 @@ def initConfig(): dir_home = ns_config_local['dir_home'] dir_home_user = ns_config_local['dir_home_user'] file_system_readme = ns_config_local['file_system_readme'] + dir_pub_class = ns_config_local['dir_home'] + '/base/class' + dir_pub_dsets = ns_config_local['dir_home'] + '/base/datasets' sysout.info(TAG, 'init system config successed! ') diff --git a/project/main/config/ws_conf.json b/project/main/config/ws_conf.json new file mode 100644 index 0000000..c2b641d --- /dev/null +++ b/project/main/config/ws_conf.json @@ -0,0 +1,4 @@ +{ + "ws_host": "127.0.0.1", + "ws_port": "8101" +} diff --git a/project/main/manager/__pycache__/__init__.cpython-37.pyc b/project/main/manager/__pycache__/__init__.cpython-37.pyc new file mode 100644 index 0000000..41d0e1d Binary files /dev/null and b/project/main/manager/__pycache__/__init__.cpython-37.pyc differ diff --git a/project/main/manager/__pycache__/fileManager.cpython-36.pyc b/project/main/manager/__pycache__/fileManager.cpython-36.pyc index b582bf8..835ee74 100644 Binary files a/project/main/manager/__pycache__/fileManager.cpython-36.pyc and b/project/main/manager/__pycache__/fileManager.cpython-36.pyc differ diff --git a/project/main/manager/__pycache__/fileManager.cpython-37.pyc b/project/main/manager/__pycache__/fileManager.cpython-37.pyc new file mode 100644 index 0000000..66dda7b Binary files /dev/null and b/project/main/manager/__pycache__/fileManager.cpython-37.pyc differ diff --git a/project/main/manager/__pycache__/jupyter.cpython-37.pyc b/project/main/manager/__pycache__/jupyter.cpython-37.pyc new file mode 100644 index 0000000..cae68ae Binary files /dev/null and b/project/main/manager/__pycache__/jupyter.cpython-37.pyc differ diff --git a/project/main/manager/__pycache__/projectManager.cpython-36.pyc b/project/main/manager/__pycache__/projectManager.cpython-36.pyc index 00c3e7f..29036b3 100644 Binary files a/project/main/manager/__pycache__/projectManager.cpython-36.pyc and b/project/main/manager/__pycache__/projectManager.cpython-36.pyc differ diff --git a/project/main/manager/__pycache__/projectManager.cpython-37.pyc b/project/main/manager/__pycache__/projectManager.cpython-37.pyc new file mode 100644 index 0000000..0320ff7 Binary files /dev/null and b/project/main/manager/__pycache__/projectManager.cpython-37.pyc differ diff --git a/project/main/manager/__pycache__/vmManager.cpython-37.pyc b/project/main/manager/__pycache__/vmManager.cpython-37.pyc new file mode 100644 index 0000000..ae414a9 Binary files /dev/null and b/project/main/manager/__pycache__/vmManager.cpython-37.pyc differ diff --git a/project/main/manager/fileManager.py b/project/main/manager/fileManager.py index 9dc146d..a0f2a25 100644 --- a/project/main/manager/fileManager.py +++ b/project/main/manager/fileManager.py @@ -8,6 +8,7 @@ from base import sysout from config import config from manager import jupyter +import asyncio TAG = 'filemanager' @@ -47,6 +48,10 @@ def getAllFiles(dir, containSystem=True): }) return result +def listFile(dir): + if dir == None or (not os.path.exists(dir)): + return None + return os.listdir(dir) # # get files number of one dir @@ -120,7 +125,7 @@ def setTimeout(num, popen): def getDirNumber2(dir): cmd = 'ls -l ' + str(dir) + ' |grep "^d"|wc -l' n = os.popen(cmd).read() - return n + return str(n).strip() # # get only one file name of the dir @@ -205,7 +210,8 @@ def getFilesInfoOfPath(dir): timeModify = str(os.path.getmtime(filePath)).split(".")[0] timeCreate = str(os.path.getctime(filePath)).split(".")[0] size = 0 - if os.path.isfile(filePath) or (not str(c).startswith(".") and "." in str(c)): + # if os.path.isfile(filePath) or (not str(c).startswith(".") and "." in str(c)): + if os.path.isfile(filePath): # is a file # files.append(c) # get file's properties @@ -302,10 +308,13 @@ def getFilesInfoOfPath(dir): # # # -def getDirSize(dir): +async def getDirSize(dir): size = 0 for root, dirs, files in os.walk(dir): - size += sum([os.path.getsize(os.path.join(root, name)) for name in files]) + try: + size += sum([os.path.getsize(os.path.join(root, name)) for name in files]) + except: + pass mSizeDict[dir] = size return size @@ -398,16 +407,16 @@ def makeDir(dir): return (0, 'Directory form not support, should be absolute path !') else: try: - shell.execute("mkdir " + dir) + os.makedirs(dir) return (1, 'Create successed !') except Exception as e: return (0, 'Create directory failed, ' + str(e)) def createFile(file): - if dir == None: + if file == None: return (0, 'File can not be null !') - if "/" not in str(dir): + if "/" not in str(file): return (0, 'File form not support, should be absolute path !') else: try: diff --git a/project/main/manager/projectManager.py b/project/main/manager/projectManager.py index 2e29abb..d855d87 100644 --- a/project/main/manager/projectManager.py +++ b/project/main/manager/projectManager.py @@ -4,8 +4,11 @@ from config import config, string from manager import fileManager, vmManager from system import shell +from hashlib import md5 import os import json +import threading, time +import urllib.request TAG = "projectManager" @@ -15,13 +18,43 @@ resp_err_version_create_dir_err = {'status': 0, 'result': 'system error on create new directory!'} +# +# public check method +# +def checkClassUsrPj(userId, classId, pjIds): + pathClass = config.dir_pub_class + '/' + str(classId) + print(pathClass) + if classId == None or not os.path.exists(pathClass): + return { + 'status': 0, + 'result': 'No such Class, please check the path!' + } + numPjs = fileManager.getDirNumber2(pathClass) + # size have to same + if not numPjs == str(len(pjIds)): + return { + 'status': 0, + 'result': 'Class and Project Size error! ' + str(numPjs) + ' projects in class but ' + str( + len(pjIds)) + ' project ids.' + } + pathUser = config.dir_home + '/' + str(userId) + if not os.path.exists(pathUser): + return { + 'status': 0, + 'result': "No such user, please check again!" + } + # success checked + return None + + # # check the version of pj is exists? # def checkVersion(userId, projectId, v): if v == None: return (False, resp_err_version_invlid) - maxVersion = int(fileManager.getDirNumber2(config.dir_home + config.dir_home_user + '/' + str(userId) + '/system/' + str(projectId) + '/')) + maxVersion = int(fileManager.getDirNumber2( + config.dir_home + config.dir_home_user + '/' + str(userId) + '/system/' + str(projectId) + '/')) if (v <= 0 or v > maxVersion): return (False, resp_err_version_invlid) return (True, maxVersion) @@ -44,15 +77,16 @@ def createPreProject(userId, projectId, projectName, projectType): os.makedirs(pjHome) shell.execute('cp ' + config.file_system_readme + ' ' + pjHome + '/') - #create directory for users's dataSets - pathDsets = config.dir_home + "/" + config.dir_home_user + '/' + str(userId) + '/system/datasets' + # create directory for users's dataSets + pathDsets = config.dir_home + '/' + str(userId) + '/system/datasets' if not os.path.exists(pathDsets): os.makedirs(pathDsets) dirData = userHome + '/数据集' if not os.path.exists(dirData): - os.makedirs(dirData) - shell.execute('ln -s ' + pathDsets + '/* ' + dirData + '/') + os.symlink('system/datasets',dirData) + # os.makedirs(dirData) + # shell.execute('ln -s ' + pathDsets + '/ ' + dirData + '/') path = config.dir_home_user + '/' + str(userId) + '/system/' + str( projectId) + '/1' # 1--version of pj， vesionInit=1 @@ -107,7 +141,7 @@ def createNewVersion(userId, projectId, projectName, versionCur): def runWithVm(userId, projectId, projectName, version, vmId, passwd, isoName, isoRemarks, gpu, cpu, memory, - action='start', pstartTime = None, pendTime = None): + action='start', pstartTime=None, pendTime=None): res = None # if action != None and action == 'stop': # # shutdown vm @@ -158,38 +192,36 @@ def delectProject(userId, pjId, pjName): return fileManager.deleteFile(projectPath) - # # bind dataset with project # isUnbind = True --> unbind dataset # -def bindDataWithProject(userId, projectId, version, dataIds, isUbind = False): +def bindDataWithProject(userId, projectId, version, dataIds, isUbind=False): if not isUbind: - #bind dataset + # bind dataset paths = [] for i in range(len(dataIds)): pathPj = "" pathDset = "" if config.dir_home_user != '': - pathPj = config.dir_home + config.dir_home_user + '/' + str(userId) + '/system/' + str(projectId) + '/' + str(version) + '/dataset/' + pathPj = config.dir_home + config.dir_home_user + '/' + str(userId) + '/system/' + str( + projectId) + '/' + str(version) + '/dataset/' # pathDset = config.dir_home + config.dir_home_user + '/' + str(userId) + '/system/datasets/' + str(dataIds[i]) else: - pathPj = config.dir_home + '/' + str(userId) + '/system/' + str(projectId) + '/' + str(version) + '/dataset/' + pathPj = config.dir_home + '/' + str(userId) + '/system/' + str(projectId) + '/' + str( + version) + '/dataset/' # pathDset = config.dir_home + '/' + str(userId) + '/system/datasets/' + str(dataIds[i]) # pathPj = '/data/system/' + str(projectId) + '/' + str(version) + '/dataset/' - pathDset = '/data/system/datasets/' + str(dataIds[i]) - if not os.path.exists(pathPj): - os.mkdir(pathPj) + os.makedirs(pathPj) + # pathDset = config.dir_home + config.path_dataset + '/' + str(dataIds[i]) # pathDset = config.dir_home + config.dir_home_user + '/'+str(userId) + '/system/datasets/' + str(dataIds[i]) - if not os.path.exists(pathDset): - shell.execute('mkdir '+ pathDset) - # return { - # 'status' : 0, - # 'result' : 'Dataset file not exists!' - # } + pathDset = '../../../datasets/' + str(dataIds[i]) + # if not os.path.exists(pathDset): + # os.makedirs(pathDset) + shell.execute('ln -s ' + pathDset + ' ' + pathPj) pathDsetln = 'dataset/' + str(dataIds[i]) + '/' paths.append(pathDsetln) @@ -197,23 +229,27 @@ def bindDataWithProject(userId, projectId, version, dataIds, isUbind = False): return { 'status': 1, 'result': { - 'message':"dataset bind successde!", + 'message': "dataset bind successde!", 'path': paths } } else: - #unbind dataset + # unbind dataset done = {} for i in range(len(dataIds)): path = '' path1 = '' if config.dir_home_user != '': - path = config.dir_home + config.dir_home_user + '/' + str(userId) + '/system/' + str(projectId) + '/' + str(version) + '/dataset/' + str(dataIds[i]) - path1 = config.dir_home + config.dir_home_user + '/' + str(userId) + '/system/' + str(projectId) + '/' + str(version) + '/dataset' + path = config.dir_home + config.dir_home_user + '/' + str(userId) + '/system/' + str( + projectId) + '/' + str(version) + '/dataset/' + str(dataIds[i]) + path1 = config.dir_home + config.dir_home_user + '/' + str(userId) + '/system/' + str( + projectId) + '/' + str(version) + '/dataset' else: - path = config.dir_home + '/' + str(userId) + '/system/' + str(projectId) + '/' + str(version) + '/dataset/' + str(dataIds[i]) - path1 = config.dir_home + '/' + str(userId) + '/system/' + str(projectId) + '/' + str(version) + '/dataset' + path = config.dir_home + '/' + str(userId) + '/system/' + str(projectId) + '/' + str( + version) + '/dataset/' + str(dataIds[i]) + path1 = config.dir_home + '/' + str(userId) + '/system/' + str(projectId) + '/' + str( + version) + '/dataset' # path = config.dir_home + config.dir_home_user + '/' + str(userId) + '/system/' + str(projectId) + '/' + str(version) + '/dataset/' + str(dataIds[i]) @@ -227,7 +263,7 @@ def bindDataWithProject(userId, projectId, version, dataIds, isUbind = False): havaException = True if not os.path.exists(path): - #the data not binded + # the data not binded # done[str(i)] = 'data hava not been binded before!' done[str(i)] = 1 else: @@ -239,7 +275,7 @@ def bindDataWithProject(userId, projectId, version, dataIds, isUbind = False): if not done[str(k)] == 1: success = False - if success : + if success: return { 'status': 1, 'result': "unbind datasets success!" @@ -256,15 +292,15 @@ def deleteDataset(userId, dataId): path = '' pathDsetFile = '' - #first unbind dataset of all project + # first unbind dataset of all project if config.dir_home_user != '': path = config.dir_home + config.dir_home_user + '/' + str(userId) + '/system/' - pathDsetFile = config.dir_home + config.dir_home_user + '/' + str(userId) + '/system/datasets/' + str(dataId) + pathDsetFile = config.dir_home + config.dir_home_user + '/' + str(userId) + '/system/datasets/' + str( + dataId) else: path = config.dir_home + '/' + str(userId) + '/system/' pathDsetFile = config.dir_home + '/' + str(userId) + '/system/datasets/' + str(dataId) - content = fileManager.getAllFiles(path) for i in range(len(content)): # is project's dataset dir ? == 1. 'dataset/dataId' is in pwd and 2. len(pwds) == 8 @@ -278,7 +314,7 @@ def deleteDataset(userId, dataId): version = pwds[5] bindDataWithProject(userId, projectId, version, [dataId], True) - #second delete file + # second delete file (code, msg) = fileManager.deleteFile(pathDsetFile) if code == 1: return { @@ -298,6 +334,197 @@ def deleteDataset(userId, dataId): } +# +# @param pjVersion : default 1 +# +def copyClassProject(userId, pjIds): + # check = checkClassUsrPj(userId, classId, pjIds) + # if not check == None: + # return check + # check done, start copy + if not os.path.exists(config.dir_home + "/" + str(userId) + "/system/datasets/"): + os.makedirs(config.dir_home + "/" + str(userId) + "/system/datasets/") + for pjid in pjIds: + version = 1 # default project version = 1 + fromPj = config.dir_home + "/0/system/" + str(pjid["baseId"]) + toPj = config.dir_home + "/" + str(userId) + "/system/" + str(pjid["id"]) + try: + if (os.path.exists(toPj)): + fromPj = fromPj + "/*" + shell.execute('cp -r ' + fromPj + ' ' + toPj) + if os.path.exists(toPj+"/1/dataset/"): + rename(toPj+"/1/dataset/",userId) + sysout.info(TAG,"copyClassProject success") + except Exception as e: + sysout.err(TAG, str(e)) + return { + 'code': 0, + 'msg': 'init project failed, cause ' + str(e) + } + return { + 'status': 1, + 'result': 'init project success!' + } + +def rename(path,userId): + filelist = os.listdir(path) #该文件夹下的所有文件 + for file in filelist: + Olddir = os.path.join(path,file) + if os.path.isfile(Olddir): + continue + filename = os.path.splitext(file)[0] + Newdir = os.path.join(path,filename) + os.rename(Olddir,Newdir) + pathDset = '../../../datasets/' + filename.replace('0-',str(userId)+'-',1) + shell.execute('ln -snf ' + pathDset + ' ' + Newdir) + shell.execute('cp -r '+ Newdir +' '+ os.path.join(path,filename.replace('0-',str(userId)+'-',1))) + +def copyClassDatasets(coursewareId,userId, datasets,projectId,datasetId): + for dataset in datasets: + fromPj = config.dir_home + "/0/system/datasets/" + dataset + toPj = config.dir_home + "/" + str(userId) + "/system/datasets/"+ str(userId) + str(dataset[1:]) + try: + if os.path.exists(toPj): + shell.execute('rm -rf ' + toPj) + if not os.path.exists(fromPj): + sysout.err('fromPj not exit') + data = {'status': '0','userId':userId,'courseId':coursewareId,'projectId':projectId,'datasetId':datasetId, 'result': 'No such Class, please check the path!'} + httpServer('http://'+config.ns_host_pub+ ':8080/WeCloud/dlCourseware/copyDatasetsStatus',data) + return + shell.execute('cp -r ' + fromPj + ' ' + toPj) + sysout.info(TAG,"copyClassDatasets success") + except Exception as e: + sysout.err(TAG, str(e)) + data = {'status': '0','userId':userId,'courseId':coursewareId,'projectId':projectId,'datasetId':datasetId,'result':str(e)} + httpServer('http://'+config.ns_host_pub+ ':8080/WeCloud/dlCourseware/copyDatasetsStatus',data) + return + data = {'status': '1','userId':userId,'courseId':coursewareId,'projectId':projectId,'datasetId':datasetId,'result': 'copy success!!'} + httpServer('http://'+config.ns_host_pub+ ':8080/WeCloud/dlCourseware/copyDatasetsStatus',data) + return + +def bindFileToDataset(userId,files,dir): + datasetPath = config.dir_home+"/"+str(userId)+"/system/datasets/"+str(dir)+'/' + deletePaths = [] + if not os.path.exists(datasetPath): + os.makedirs(datasetPath) + try: + for file in files: + path = file['path'] + name = file['name'] + os.symlink('../../..'+path,datasetPath+name) + deletePaths.append(datasetPath+name) + except FileExistsError as e1: + sysout.err(TAG, str(e1)) + for deletePath in deletePaths: + if os.path.exists(deletePath): + os.remove(deletePath) + return { + 'status': 0, + 'result': '绑定失败，存在同名文件！' + } + except Exception as e: + sysout.err(TAG, str(e)) + for deletePath in deletePaths: + if os.path.exists(deletePath): + os.remove(deletePath) + return { + 'status': 0, + 'result': 'bind file to dataset failed, cause ' + str(e) + } + + return { + 'status': 1, + 'result': 'bind file to dataset success!' + } + +def httpServer(url,values): + headers = { + 'User-Agent': 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)', + 'Content-Type': 'application/json' + } + data = json.dumps(values) + data = bytes(data, 'utf8') + request = urllib.request.Request(url, data, headers) + html = urllib.request.urlopen(request).read().decode('utf-8') + print(html) +# +# return the data copy process by the websocket +# + + + +async def startProcessCounting(userId, classId, binds, path_c, path, numDsetCur, numDsetTotal, sizeTotal, sizeDone, + websocket): + sysout.info(TAG, "start copy dset " + str(numDsetCur) + '/' + str(numDsetTotal)) + process = str(numDsetCur) + '/' + str(numDsetTotal) + # todo cal numCur = ? + # numDsetCur = ? + + rep_proocess = { + 'userId': userId, + 'classId': classId, + 'binds': binds, + 'sizeTotal': sizeTotal, + 'sizeDone': sizeDone, + 'numDsetTotal': numDsetTotal, + 'numDsetCur': numDsetCur, + 'process': process, + + } + print('ws_send:') + print(rep_proocess) + await websocket.send(json.dumps(rep_proocess)) + shell.execute('cp -r ' + path_c + '/* ' + path + '/') + print('copy done') + # while not websocket == None and numDsetCur <= numDsetTotal: + # # process = 0.00 + # process = str(numDsetCur) + '/' + str(numDsetTotal) + # # todo cal numCur = ? + # # numDsetCur = ? + # + # rep_proocess = { + # 'userId': userId, + # 'classId': classId, + # 'binds': binds, + # 'sizeTotal': sizeTotal, + # 'sizeDone': sizeDone, + # 'numDsetTotal': numDsetTotal, + # 'numDsetCur': numDsetCur, + # 'process': process, + # + # } + # await websocket.send(json.dumps(rep_proocess)) + # + # if numDsetCur == numDsetTotal: + # break + # else: + # time.sleep(0.25) # 1/4 of second + # continue + + +class ProcessLisener(threading.Thread): + def __init__(self, userId, classId, binds, path_c, path, numDsetCur, numDsetTotal, sizeTotal, sizeDone, websocket): + threading.Thread.__init__(self) + # self.threadID = threadId + self.name = 'nb_ws_processListener' + self.userId = userId + self.classId = classId + self.binds = binds + self.path_c = path_c + self.path = path + self.numDsetTotal = numDsetTotal + self.numDseCur = numDsetCur + self.sizeTotal = sizeTotal + self.sizeDone = sizeDone + self.websocket = websocket + + def run(self): + t = threading.current_thread() + sysout.info(TAG, 'websocketServer is stating at thread %s - %s' % (t.threadID, t.name)) + startProcessCounting(self.userId, self.classId, self.binds, + self.path_c, self.path, self.numDsetCur, + self.numDsetTotal, self.sizeTotal, self.sizeDone, self.websocket) + # # # cmd:path_dataset @@ -317,4 +544,4 @@ def deleteDataset(userId, dataId): # isoRemarks='remaraks test', gpu='geforce gtx 1070', cpu='i7-7700k', memory='16') # res = delectProject(2, 12, 'ddd') -# print(res) +# print(res) \ No newline at end of file diff --git a/project/main/server/__pycache__/httpServer.cpython-36.pyc b/project/main/server/__pycache__/httpServer.cpython-36.pyc index a6a6b21..6621039 100644 Binary files a/project/main/server/__pycache__/httpServer.cpython-36.pyc and b/project/main/server/__pycache__/httpServer.cpython-36.pyc differ diff --git a/project/main/server/__pycache__/httpServer.cpython-37.pyc b/project/main/server/__pycache__/httpServer.cpython-37.pyc new file mode 100644 index 0000000..760a032 Binary files /dev/null and b/project/main/server/__pycache__/httpServer.cpython-37.pyc differ diff --git a/project/main/server/__pycache__/websocketServer.cpython-36.pyc b/project/main/server/__pycache__/websocketServer.cpython-36.pyc new file mode 100644 index 0000000..201110d Binary files /dev/null and b/project/main/server/__pycache__/websocketServer.cpython-36.pyc differ diff --git a/project/main/server/__pycache__/websocketServer.cpython-37.pyc b/project/main/server/__pycache__/websocketServer.cpython-37.pyc new file mode 100644 index 0000000..87ac15e Binary files /dev/null and b/project/main/server/__pycache__/websocketServer.cpython-37.pyc differ diff --git a/project/main/server/httpServer.py b/project/main/server/httpServer.py index 577514e..da201ee 100644 --- a/project/main/server/httpServer.py +++ b/project/main/server/httpServer.py @@ -5,7 +5,10 @@ from wsgiref.simple_server import make_server import json import threading +import _thread import asyncio +import os +from os.path import join, getsize from base import sysout from manager import projectManager @@ -14,7 +17,6 @@ TAG = 'httpServer' - # global response data resp_err_params = {'status': '0', 'result': 'request params form error!'} @@ -22,15 +24,18 @@ class NotebookHttpServer(threading.Thread): - def __init__(self, threadId, name): + def __init__(self, threadId, name, loop): threading.Thread.__init__(self) self.threadID = threadId self.name = name + self.loop = loop + mLoop = self.loop def run(self): t = threading.current_thread() - sysout.info(TAG, 'NotebookHttpServer is stating at thread %s - %s' %(t.threadID, t.name)) - run() + sysout.info(TAG, 'NotebookHttpServer is stating at thread %s - %s' % (t.threadID, t.name)) + run(self.loop) + ## # module -- project @@ -114,7 +119,7 @@ def runWithVm(request_body): gpu = None cpu = None memory = None - action1 = 'start' #default : start + action1 = 'start' # default : start pstartTime = None pendTime = None try: @@ -158,7 +163,7 @@ def getMyFiles(request_body): return str(resp_err_params) + str(e) home = config.dir_home + '/' + str(userId) - res = fileManager.getUserHome(home) + res = fileManager.getUserHome(home) return { "status": 1, "result": res @@ -187,6 +192,7 @@ def bindDataWithProject(request_body): else: return projectManager.bindDataWithProject(userId, projectId, version, dataIds, isUbind) + def deleteDataset(request_body): userId = None dataId = None @@ -200,6 +206,7 @@ def deleteDataset(request_body): else: return projectManager.deleteDataset(userId, dataId) + # # get all files & dirs info of the current path # @@ -214,15 +221,16 @@ def getFilesInfoOfPath(request_body): res = fileManager.getFilesInfoOfPath(path) if res == None: return { - 'status':0, - 'result':"File or directory not found!" + 'status': 0, + 'result': "File or directory not found!" } else: return { - 'status':1, - 'result':res + 'status': 1, + 'result': res } + # # rename file or dir # @@ -236,8 +244,9 @@ def rename(request_body): return str(resp_err_params) + str(e) return fileManager.rename(src, dst) + # -#delete file or dir +# delete file or dir # def deleteFile(request_body): path = None @@ -251,6 +260,7 @@ def deleteFile(request_body): 'result': res } + def deleteFiles(request_body): paths = None try: @@ -265,6 +275,7 @@ def deleteFiles(request_body): 'result': result } + def makeDir(request_body): dir = None try: @@ -277,6 +288,7 @@ def makeDir(request_body): 'result': res } + def createFile(request_body): file = None try: @@ -289,6 +301,7 @@ def createFile(request_body): 'result': res } + def moveFile(request_body): file = None dir = None @@ -303,6 +316,92 @@ def moveFile(request_body): 'result': res } + +# public classes dir: +# /notebook/storage/base/class/classIds/projectIds/version/files... +# copy class's projects and files to user's pj home +# +def copyClassProject(request_body): + userId = None + projectIds = [] + # version = 1 #default project version = 1 + try: + userId = request_body['userId'] + projectIds = request_body['projectIds'] + except Exception as e: + print(str(e)) + return str(resp_err_params) + str(e) + return projectManager.copyClassProject(userId, projectIds) + + +# public datasets dir: +# /notebook/storage/base/datasets/dataIds +# +def copyClassDataset(request_body): + coursewateId = None + projectId = None + datasetId = None + userId = None + datasets = [] + # version = 1 #default project version = 1 + try: + userId = request_body['userId'] + if ('coursewareId' in request_body.keys()): + coursewateId = request_body['coursewareId'] + if ('projectId' in request_body.keys()): + projectId = request_body['projectId'] + if ('datasetId' in request_body.keys()): + datasetId = request_body['datasetId'] + datasets = request_body['datasets'] + except Exception as e: + return str(resp_err_params) + str(e) + # projectManager.copyClassDatasets(coursewateId,userId,datasets) + _thread.start_new_thread( projectManager.copyClassDatasets, (coursewateId,userId,datasets,projectId,datasetId, ) ) + return { + 'status': 1, + 'result': 'start copying datasets!' + } + + +# +# reset some onr or some pj in class +# 1. delete all versions +# 2. hold all versions +# +def resetClassProject(request_body): + projets = [] + type = request_body['type'] + userId = request_body['userId'] + project = request_body['projectIds'] + projets.append(project) + if (type == 'all'): + # reset all pj of this class + projectManager.delectProject(userId,project['id'],None) + elif (type == 'one'): + projectManager.delectProject(userId,str(project['id'])+"/1",None) + return projectManager.copyClassProject(userId,projets) + + + +async def getDirSize(request_body): + userId = request_body['userId'] + home = config.dir_home + '/' + str(userId) + # dir = '/notebook/storage/' + str(userId) + size = 0 + size = await fileManager.getDirSize(home) + return { + 'status': 1, + 'result': size + } + +def bindFileToDataset(request_body): + try: + files = request_body['files'] + dir = request_body['dir'] + userId = request_body['userId'] + except Exception as e: + return str(resp_err_params) + str(e) + return projectManager.bindFileToDataset(userId,files,dir) # # # @@ -340,7 +439,6 @@ def praseData(request_body): # futrue = asyncio.ensure_future(task) # loop.run_until_complete(futrue) # return futrue.result() - elif action == 'rename': return rename(request_body) elif action == 'deleteFile': @@ -353,9 +451,27 @@ def praseData(request_body): return createFile(request_body) elif action == 'moveFile': return moveFile(request_body) + + elif action == 'initClass': + return copyClassProject(request_body) + elif action == 'getDirSize': + loop = asyncio.get_event_loop() + return loop.run_until_complete(getDirSize(request_body)) + # loop.close() + # return a + # return getDirSize(request_body, loop) + elif action == 'copyClassProject': + return copyClassProject(request_body) + elif action == 'copyClassDataset': + return copyClassDataset(request_body) + elif action == 'resetClassProject': + return resetClassProject(request_body) + elif action == 'bindFileToDataset': + return bindFileToDataset(request_body) else: return {'status': 0, 'result': 'request & params not support!!!'} + # server def application(environ, start_response): # 定义请求的类型和当前请求成功的code @@ -388,12 +504,19 @@ def application(environ, start_response): return [result] -def run(): +def run(loop): + asyncio.set_event_loop(loop) + # httpd = HTTPServer(mServer, AmiHTTPServer) mPort = config.ns_port_http mHost = config.ns_host mServer = (mHost, mPort) httpd = make_server(mHost, mPort, application) # sysout.info(TAG, 'http server is running on ' + str(mServer)) - sysout.info(TAG, "The server now is running on %s in [%s] mode!"%(str(mServer),config.system['mode'])) + sysout.info(TAG, "\033[22;32;40m【200 SUCCESS】\033[0m" + " The http_server is now running on %s in [%s] mode!" % ( + str(mServer), config.system['mode'])) httpd.serve_forever() + + # loop.run_until_complete(httpd) + # loop.run_until_complete(asyncio.wait(httpd)) + # loop.run_forever() \ No newline at end of file diff --git a/project/main/server/websocketServer.py b/project/main/server/websocketServer.py new file mode 100644 index 0000000..615ec42 --- /dev/null +++ b/project/main/server/websocketServer.py @@ -0,0 +1,160 @@ +import asyncio +import websockets +import websocket +import json +import time +import threading + +from base import sysout +from config import config +from manager import projectManager + +TAG = 'webSocketServer' + +# global response data +resp_err_params = {'status': '0', 'result': 'request params form error!'} + +# +# init config +# +with open("config/ws_conf.json", 'r') as conf: + global load_dict + load_dict = json.load(conf) + +sysout.info(TAG, 'init ws config... ') +sysout.info(TAG, str(load_dict)) +host = load_dict['ws_host'] +port = load_dict['ws_port'] +conf.close() +sysout.info(TAG, "init ws config done, ws_host = " + str((host, port))) + + +class webSocketServer(threading.Thread): + + def __init__(self, threadId, name, loop): + threading.Thread.__init__(self) + self.threadID = threadId + self.name = name + self.loop = loop + + def run(self): + t = threading.current_thread() + sysout.info(TAG, 'websocketServer is stating at thread %s - %s' % (t.threadID, t.name)) + run(self.loop) + +def onProcess(numCur, numTotal): + sysout.info('ws.onProcess()', 'cur:'+str(numCur) + ' / total:'+str(numTotal)) + return (numCur, numTotal) + +# +# main func +# +async def onServer(websocket, path): + # sysout.info(TAG, "ws_server is start running... ") + requests = await websocket.recv() + requests = json.loads(requests) + sysout.info(TAG, "ws_server get request >> " + str(requests)) + + try: + if requests == None: + resp = { + 'status': 0, + 'result': 'Request body can not be empty!' + } + await websocket.send(json.dumps(resp)) + websocket.close() + else: + # praseRequest(websocket, requests) + # copy class's datasets + if requests['action'] == 'copyClassData': + # copyClassData(websocket) + await copyClassData(requests, websocket) + + # for i in range(101): + # done = i == 100 + # resp = { + # 'process': str((i / 100) * 100) + '%', + # 'done': done + # } + # print(resp) + # await websocket.send(json.dumps(resp)) + # if done: + # # websocket.close() + # break + # time.sleep(0.05) + + elif requests['action'] == 'xxx?': + pass + + else: + msg_err = 'Request [' + requests + '] is not not allowed!' + resp = { + 'status': 0, + 'result': msg_err + } + await websocket.send(json.dumps(resp)) + # websocket.close() + + except Exception as e: + sysout.err(TAG, "ws_server Exception: " + str(e)) + finally: + sysout.info(TAG, "ws_connection for -- " + str(requests) + " -- is closed.") + websocket.close() + pass + + +def run(loop): + # asyncio.get_event_loop_policy().set_event_loop(loop) + asyncio.set_event_loop(loop) + + start_server = websockets.serve(onServer, host, port) + sysout.info(TAG, "\033[22;32;40m【200 SUCCESS】\033[0m" + "The ws_server is now running on %s in [%s] mode!" % ( + str((host, port)), config.system['mode'])) + loop.run_until_complete(start_server) + loop.run_forever() + + +# def praseRequest(websocket, requests): + + +async def copyClassData(request, websocket): + await websocket.send(json.dumps({ + 'tips':'starting copyClassDataset...' + })) + userId = None + classId = None, + # projectId = None + binds = [] + try: + userId = request['userId'] + classId = request['classId'] + # projectId = request['projectId'] + binds = request['binds'] + + except Exception as e: + sysout.err(TAG, str(e)) + return resp_err_params + + # for test + # show the connection of project and there's datasets + # binds = [ + # { + # 'classPjId': 1, # project id of class + # 'pjId': 100, # user's project id + # 'version': 1, # version of pj; default 1 + # 'dsetIds': [100 - 1, 100 - 2, 100 - 3] # dataset id (same both user's & class) + # }, + # { + # 'classPjId': 2, + # 'pjId': 101, + # 'version': 1, # version of pj; default 1 + # 'dsetIds': [1] + # } + # ] + + # projectManager.copyClassDatasets(userId, classId, binds, onProcess, websocket) + res = await projectManager.copyClassDatasets(userId, classId, binds, onProcess, websocket) + if not res == None: + await websocket.send(json.dumps(res)) + if res['status'] == 0: + websocket.close() \ No newline at end of file diff --git a/project/main/startup.py b/project/main/startup.py index 010d44b..acbf136 100644 --- a/project/main/startup.py +++ b/project/main/startup.py @@ -4,15 +4,28 @@ from config import config, string from server import httpServer from manager import fileManager +import threading +import asyncio TAG = "NotebookServer" -warning = '\n ---- \n System Warning:\n ---- \n Before u start the NotebookServer, u have to make sure that u have already installed the module "Jupyter Notebook"! \n' + " Even though, there'll be some error when u use it !! \n ---- ---- ---- ---- \n" +warning = '\n ---- ' \ + '\n System Warning:\n ' \ + '---- \n ' \ + 'Before u start the NotebookServer, u have to make sure that u have already installed the module "Jupyter Notebook"! \n' + \ + " Even though, there'll be some error when u use it !! \n" \ + " ---- ---- ---- ---- \n" +Warning = "\033[0;37;43m" + warning + "\033[0m" -if __name__ == '__main__': + +def initConfig(): config.initConfig() - sysout.info('', warning) + sysout.info('', "\033[0;37;43m" + warning + "\033[0m") + + +def startHttpServer(): httpServer.run() + # t = threading.Thread(target=httpServer.run(), name='NotebookHttpServer') # thread.start() # thread.join() @@ -22,3 +35,25 @@ # notebookServer = httpServer.NotebookHttpServer(threadId=1, name='NotebookHttpServer') # notebookServer.start() # notebookServer.join() + + +def startWebsocket(): + from server import websocketServer + websocketServer.run() + # ws_server = websocketServer.webSocketServer(threadId=2, name='nbWsServer', loop=asyncio.new_event_loop()) + # ws_server.start() + # ws_server.join() + + +if __name__ == '__main__': + initConfig() + # startWebsocket() + # startHttpServer() + + pool = [] + pool.append(httpServer.NotebookHttpServer(threadId=1, name='NotebookHttpServer', loop=asyncio.new_event_loop())) + from server import websocketServer + pool.append(websocketServer.webSocketServer(threadId=2, name='nbWsServer', loop=asyncio.new_event_loop())) + for t in pool: + t.start() + # t.join() diff --git a/project/main/system/__pycache__/__init__.cpython-37.pyc b/project/main/system/__pycache__/__init__.cpython-37.pyc new file mode 100644 index 0000000..f8a5a75 Binary files /dev/null and b/project/main/system/__pycache__/__init__.cpython-37.pyc differ diff --git a/project/main/system/__pycache__/shell.cpython-36.pyc b/project/main/system/__pycache__/shell.cpython-36.pyc index df81abb..775a513 100644 Binary files a/project/main/system/__pycache__/shell.cpython-36.pyc and b/project/main/system/__pycache__/shell.cpython-36.pyc differ diff --git a/project/main/system/__pycache__/shell.cpython-37.pyc b/project/main/system/__pycache__/shell.cpython-37.pyc new file mode 100644 index 0000000..4fd3899 Binary files /dev/null and b/project/main/system/__pycache__/shell.cpython-37.pyc differ diff --git a/project/main/utils/__pycache__/__init__.cpython-37.pyc b/project/main/utils/__pycache__/__init__.cpython-37.pyc new file mode 100644 index 0000000..079a6d0 Binary files /dev/null and b/project/main/utils/__pycache__/__init__.cpython-37.pyc differ diff --git a/project/main/utils/__pycache__/socketUtils.cpython-37.pyc b/project/main/utils/__pycache__/socketUtils.cpython-37.pyc new file mode 100644 index 0000000..b6b6495 Binary files /dev/null and b/project/main/utils/__pycache__/socketUtils.cpython-37.pyc differ diff --git a/project/test/__init__.py b/project/test/__init__.py index 5d47134..ce122d4 100644 --- a/project/test/__init__.py +++ b/project/test/__init__.py @@ -1,2 +1,13 @@ import os -print(os.getcwd()) \ No newline at end of file +import datetime +import time +# print(os.getcwd()) + +# print(time.asctime( time.localtime(time.time()) )) +# print(time.strftime("%Y.%m.%d-%H:%M:%S", time.localtime())) + + +dsetIds = ['sys-dset1', 'sys-dset2'] + +for d in dsetIds: + print(dsetIds.index(d)) \ No newline at end of file diff --git a/project/test/process.py b/project/test/process.py new file mode 100644 index 0000000..cf653d9 --- /dev/null +++ b/project/test/process.py @@ -0,0 +1,21 @@ + +arrDsets = [] +for d in pjs.dsets: + if not d in arrDsets: + arrDsets.append(d) +numDsetsTotal = len(arrDsets) # size of the datasets that need to be copy + + +for p in projects: + numCurPj = projects.index(p) + 1 + + + for d in dsets: + numCurDset = dsets.index(d) + 1 + + path = pUser + '/system/datasets' + d.Id + if not os.path.exists(path): + #start copying + startProcessThreadForDataset() + shell.exec(cp) + diff --git a/project/test/syncTask.py b/project/test/syncTask.py new file mode 100644 index 0000000..a99b030 --- /dev/null +++ b/project/test/syncTask.py @@ -0,0 +1,14 @@ +import threading +import time +def onProcess(id, p): + print('onProcess id='+str(id) + ' p='+str(p)) + +def test(ids, onProcess): + print('test') + for id in ids: + print("index: " + str(ids.index(id))) + onProcess(id, id/len(ids)) + time.sleep(0.1) + +if __name__ == '__main__': + test([1,2,3,4], onProcess) \ No newline at end of file diff --git a/project/test/wsClient.py b/project/test/wsClient.py new file mode 100644 index 0000000..2774ddb --- /dev/null +++ b/project/test/wsClient.py @@ -0,0 +1,52 @@ +import asyncio +import websockets +import json + +params = { + 'action': 'copyClassData', + 'userId': 396, + 'classId': 1, + 'binds': [ + { + 'classPjId': 10, # project id of class + 'pjId': 3, # user's project id + 'version': 1, # version of pj; default 1 + 'dsetIds': ['sys-dset-1', 'sys-dset-2'] # dataset id (same both user's & class) + }, + { + 'classPjId': 11, + 'pjId': 4, # user's project id + 'version': 1, + 'dsetIds': ['sys-dset-1'] + } + ] +} + + +async def hello(uri): + async with websockets.connect(uri) as websocket: + # print(websocket.is_alive()) + # print(websocket.isAlive()) + # print(websocket.close()) + await websocket.send(json.dumps(params)) + # res = await websocket.recv() + # print(res) + try: + while True: + res = await websocket.recv() + res = json.loads(res) + print(res) + # if res['done'] == True: + # websocket.close() + # print('Procress to 100% Success!') + # break + except Exception as e: + print(e) + websocket.close() + + +# main +asyncio.get_event_loop().run_until_complete(hello('ws://localhost:8101')) + +# while True: +# asyncio.get_event_loop().run_until_complete(hello('ws://localhost:8765')) diff --git a/readme-ln b/readme-ln new file mode 100644 index 0000000..bbc7432 --- /dev/null +++ b/readme-ln @@ -0,0 +1,16 @@ +# +# This isfor u to use the NotebookServer +# + +1. where to place: + /notebook/ + +2. start cmd: + python3 /notebook/NotebookServer/project/main/startup.py + +3. repo: + current new branch : class + you have to checkout to a new branch for u users + + +