From 9935cc566ec6978fe066c385e447dd54e28a90bc Mon Sep 17 00:00:00 2001 From: JerryYin Date: Wed, 31 Oct 2018 14:59:45 +0800 Subject: [PATCH 01/42] unbind dataset done --- .idea/workspace.xml | 302 +++++++++--------- .../config/__pycache__/config.cpython-36.pyc | Bin 1822 -> 1691 bytes project/main/config/config.py | 73 +++-- .../__pycache__/fileManager.cpython-36.pyc | Bin 2568 -> 2564 bytes .../__pycache__/jupyter.cpython-36.pyc | Bin 1069 -> 1221 bytes .../__pycache__/projectManager.cpython-36.pyc | Bin 3472 -> 4629 bytes .../__pycache__/vmManager.cpython-36.pyc | Bin 1930 -> 2006 bytes project/main/manager/jupyter.py | 20 +- project/main/manager/projectManager.py | 78 ++++- project/main/manager/vmManager.py | 8 +- .../__pycache__/httpServer.cpython-36.pyc | Bin 3746 -> 4356 bytes project/main/server/httpServer.py | 38 ++- project/test/codeTest.py | 19 ++ 13 files changed, 330 insertions(+), 208 deletions(-) diff --git a/.idea/workspace.xml b/.idea/workspace.xml index 84a7c42..a4b7e33 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -3,8 +3,12 @@ - + + + + + - + + - + - - + + - - + + @@ -534,30 +550,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -629,13 +604,6 @@ - - - - - - - @@ -755,13 +723,6 @@ - - - - - - - @@ -823,13 +784,6 @@ - - - - - - - @@ -844,13 +798,6 @@ - - - - - - - @@ -872,82 +819,125 @@ - + + - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - + + - + - - + + + + + - + - - + + + + + - + - - - - - + + - + - - + + - + - - + + - + + + + + + + + - - + + - - - + + + diff --git a/project/main/config/__pycache__/config.cpython-36.pyc b/project/main/config/__pycache__/config.cpython-36.pyc index a364eedfe44538449a7ced9480624e388f1c6505..607aa0c88a9bfe2d1b73b2288ec3e984022f1db7 100644 GIT binary patch delta 373 zcmYk%yGjE=6b9g#-A!DR?B<@$)msV;7#EFNXk%fk7IrHVSI({xl*FC&LJ({e1gkKO zAovotg00{K2)6d(Bls&8nZx|wIm|FHkLH6>Zy84H?fZS_#bE4{4J{S@8n$>I)Yq@# zl(3j%kl~nQI!P~1Nz5sjf@zqB8JK}tn1wl*gL#;T1z3OUP^x-CS9SFV$`w#S delta 490 zcmYk&y-UMD6aesCl4@F#wzjpYwY6zGDQM9;NCgoTH(lH!5@O!f`jMRE;s;q26!(%* z5Zna^2mb;804E1m{Wk<(;|Jbx@Auxpy}RT_yG=LSnwC=UU$QD; zTs>D!4OBBtHPCdy%s|@ALdG1yI*WA<9YOPG4lSU0bQCS1MRXL^(IQ$xb#x3Zp=B6@ z@|9qY!#GsX3RGbNtwIe9bYckVo5EzH|G`Nkx|7N&j|Fsxv4e1GP&&V5iwDfNcj=oD zs;wQH)0R(daA~-}d9TZj=@ny|89sHG=TRpxjyd(0dyEI3?b41T~)bVe?s6WJn{(W|_jzaIP}5raT{9plDiEm}+X^&B3ydd#I( z+-q=&ss8N&J&B$YGfmxb!HVBzx$M{rte+1sli!<1rx1N6V5x(hWr8y+ix{WkoD`&3 VNfwnPX0k$XivOad6ctI4z5qK-d{O`a diff --git a/project/main/config/config.py b/project/main/config/config.py index 529ac35..2986706 100644 --- a/project/main/config/config.py +++ b/project/main/config/config.py @@ -25,43 +25,40 @@ # ----------- RELEASE ------------ -ns_host = '172.16.59.99' #_release server cloudyotech.com -ns_port_http = 8100 -ns_port = 8888 -ns_doname = 'https://g.cloudyotech.com/notebook' -# ns_doname = 'http://' + ns_host + ":" + str(ns_port) - -# system home dir -dir_home_release = '/notebook/storage' -dir_home = dir_home_release - -# dir_home_user = '/users' -dir_home_user = '' -# user's project dir: homeDir/users/userId/projectId/version/xxx-nb - -file_system_readme = dir_home +'/base/readme' - - - -# ----------- DEV ------------ -# ns_host = '172.16.3.254' +# ns_host = '172.16.59.99' #_release server cloudyotech.com # ns_port_http = 8100 # ns_port = 8888 -# ns_doname = 'https://dev.dongxicc.cn/notebook' -# ## ns_doname = 'http://' + ns_host + ":" + str(ns_port) +# ns_doname = 'https://g.cloudyotech.com/notebook' +# # ns_doname = 'http://' + ns_host + ":" + str(ns_port) # # # system home dir -# # dir_home_dev = '/root/notebook/storage' -# dir_home_release = '/notebook/storage/users' +# dir_home_release = '/notebook/storage' # dir_home = dir_home_release # -# dir_home_user = '/users' -## dir_home_user = '' +# # dir_home_user = '/users' +# dir_home_user = '' # # user's project dir: homeDir/users/userId/projectId/version/xxx-nb +# # file_system_readme = dir_home +'/base/readme' +# ----------- DEV ------------ +ns_host = '172.16.3.254' +ns_port_http = 8100 +ns_port = 8888 +ns_doname = 'https://dev.dongxicc.cn/notebook' +## ns_doname = 'http://' + ns_host + ":" + str(ns_port) + +# system home dir +dir_home = '/notebook/storage' +# dir_home_user = '/users' +dir_home_user = '' +# user's project dir: homeDir/users/userId/projectId/version/xxx-nb +file_system_readme = dir_home +'/base/readme' + + + # ----------- localohst ------------ # ns_host = '192.168.188.105' #_localhost # ns_port_http = 8100 @@ -69,10 +66,7 @@ # ns_doname = 'http://' + ns_host + ":" + str(ns_port) # # ## system home dir -# dir_home_localhost = '/Users/jerryyin/workspace/notebook' -# dir_home_dev = '/Users/jerryyin/workspace/notebook/storage' -# dir_home = dir_home_dev -# +# dir_home = '/Users/jerryyin/workspace/notebook/storage-dev' # # dir_home_user = '/users' # dir_home_user = '' # ## user's project dir: homeDir/users/userId/projectId/version/xxx-nb @@ -82,11 +76,22 @@ #cmd #sshfs for storage on StorageServer -# 'sshfs -C -o reconnect root@60.12.136.60:/sshfs/jupyter ~/workspace/notebook/storage' -# sshfs root@60.12.136.60:/sshfs/ ~/workspace/notebook/storage -# umount -f ~/workspace/notebook/storage -cmd_sshfs_mount_storage = 'sshfs -C -o reconnect user@hostname:remote_dir local_dir' +#cmd_sshfs_mount_storage = 'sshfs -C -o reconnect user@hostname:remote_dir local_dir' +# +# release +# sshfs root@60.12.136.60:/sshfs/ /notebook/storage sshfs_pwd = 'fd324;1' +# +# dev & localhost [private network] +# sshfs root@10.10.45.246:/mnt/ ~/workspace/notebook/storage +# sshfs_pwd = 'abcd#12345' +# dev2 [common network] +# sshfs root@60.12.136.61:/mnt +# sshfs_pwd = 'abcd#12345' + +# path of the user's dataset +path_dataset = '/file/datasets' + # the common base nb of the sys diff --git a/project/main/manager/__pycache__/fileManager.cpython-36.pyc b/project/main/manager/__pycache__/fileManager.cpython-36.pyc index 4c1d430923162220e25acae3434764de398b9024..80449a717b98992f4e35dcda2dc8f0f8d517b769 100644 GIT binary patch delta 299 zcmeAWX%XQx=H=!3a^qTb!*mVv{#8momytR%5XODo$i^W>lKIlqG>tY%&{bAfx)`2-Yo(j9Qa<*~_%_ zK=~|JTT2PXipQp(Taku1TZBAQZCy-{w5KRGy fenB92avi6h6xc&@Ani&ZLLEfdfC#6_$2gq;c=@8*G=H=yDboN?w)ke;ECgv)3y~$Zj!Hj~F_c6IKN=;^E?voK_U|`T>E#d%D zMVugl3q(kP2+_&wnM)aEC#$kp0Tm~(I5R3vUc!>VC_0&yHIPwlb2#f3Mn=uaJnUs! zx*+v>K;jllNo7H56h~TSPHI46Nk)+&NJbq*Xn+XK$!pmC8LcP(VV}jQGkH3Pm!}R$ z*ak>wG8Nf^*mfW`JGzUQKvJe4LJmmW;&iP@PAw?O%+J$ghPYc|vKFVUm?KCtV~C~z mSpO}y;>zNZ)ZEEcoO)7VKgohjPy`WbAi^3%I8Hvq=>!1VRD6L4vpFkA{qpAnEwg`j@C(=q(YA{^t?1E@xI~v=FpahV@ zxe!_6$`kMk+_=C4Ao@<2hjVtQw4>E~2wm-JkG-ec+GpR^@(~1`}(i3Ufpp}#NqhCGJK9i_=jL-`;&$H&ZO092wYXoaH@&)Te-+he=O zmVIKFMl+$|MA3@IxtPJ+%3%gt^kIg5Sm1}Xd3h>!ZXQCiB_VH}c=Qor?jpSI<@T|Y ztC_nH2#4^v7K?A*xI;+95Xv9o0NwJc)!%7xq0pQBo4s0nnCfC|25EjjEfT#;loCm1 zRhA*Dtb@A`0*}@>L7j5n+KejvZ{oITBz=)QA4C_KnbmYafeIz>9S5Z+BT$)-(2s$f0zpgje8QLiF`C_M#15sY4%>?)#3%4{kM9t!#bB{$!} zgRdZXv2UwR=Job@*bi7kj= zcPw`jhZ5lk_k|>`@C3)~3174@b77y5s6Cr@*W!rkmBR;_O)7Z|atviAl`3Z+Q`f_# z;MGHV8Tw|w0U{Lq`kg)t2kXev?5c!ZyG0t0atXgb@C7&q?Pq0iFQJOl5UP`D}%h9YH%R=)aHF%>+Gms+z42P4}_>Xqv zLkC;XuIKh3{!gbZ8<{u7b)2pntR)Och%!7K+NA-7E_QH>k*Zzws4AI*;R&ntIqMx6 lKrnn81A>v6rVTydHdz_oxdHKy~K*RBDx}>F(5_3u^c3;#J!R{_{~P=sD&AHn6lt((`~rQiOf-&v2iJQt(O7H`|Jdt@e75e0 z{(ZDq+R~&Uej~U>aTV|{Q$mCq|FJ_-?dZ3JBp?vJ0W6ZweoFO(h6*kfp+x`_O%y;f z#0Tji3}sz}QiMKTgiFoR#YC5HWmlPgN)bFe zbeJKqX|izwdn@Cx3Z}mBmB&geLJM6{k=nzFHNq@r3k@5AvmxVj`XsfE&aTw<`~Ru! za^lo34{Z?7IU3HYouY)Hdx|4I9cc9Cc^2%^F%VE$&X z-Q_`}XYhUh7T@-J2J1$F|E|Bf)U(T&3&{4So<>EUGHHCCu+gmN8gEBCL9XltQPy_G z`!kreNX@Sym23OEtiyBXpq)JVy!7XWw3(A?y?r3>a$dQAv=tmAoo>YOK9XDsIOo@; zSqr!32$%Pj*W}j3EjMJS2Q1EwPW-VvTdv6rao&f{hAzp6;f5*yiwOIMbx!R?xhjh+ zPMAZG8)cn2>~i@m54Q=Kqb}nvVxF}QhU48Z*h;=bMf?~y0xDIZ26w;#3$B4fDVTHu zJ{$v2p~yJ8RNn=UdT2tygDUb4+|*1kLF&vxVo}NK(p{y%W$dQH-@@O@N)wjw_50IV h&~r76yHV!5^Y%SdarvO}$JzVtO_<9L-8ZOd{R{EXnz;Y~ delta 648 zcmZ`$L2J}d5PtJsl1*M-vYTXg)782xwiLIA-31ZElUOK=poi{3S)rhPy<~M$l3ZG5 z=^>@(N)S90@h^y1kDe5-9`@?ZpWw-t*n$Op!!X}`GxG+%d9!|YeeJg6@YdJw&vrvX zevrQ&Q~18}^zG}l)37al8Jxh>mrdC^A=5zW6VeYqe0c=S8V){oXq&do;UGOsv;9{` zZ114=3wlRo>4DZ}RnN5c3690ZiaW64NGn9*dEkqD5Wfg0*) zqJcs3Bd4-<9FmM7AEV=@nZ8h76p-g}YCe;`R NWrpdR*UX!I@EeJBf;9jD diff --git a/project/main/manager/__pycache__/vmManager.cpython-36.pyc b/project/main/manager/__pycache__/vmManager.cpython-36.pyc index e1dae729375b9953de870daae23d4f1ebb64d065..e0f4b4eec7fdc7440f1710c50e702e5ea97578c8 100644 GIT binary patch delta 655 zcmYk3J!=#}7{}-3ZeM0^ch6knnRt-|L|t>yB!X2MjfGK&LaZVvnpyNV6H2~HjDXl!CECec z6P9!o%BBgsB=e*LZ;|#hVFJ@KQOQTKiJ_XmBI-{)#KhH*03y}FfRuEHso&sor-}4n z&bzVpA^*{M52QdLp+L64624}eWNeAx=BtclyUv0-pW&)8qh8Kye+XY+Y5h3S(BW^?D%!_z1^qKB9TB%X3n zUqbkZ^N0(G1aT38d#iB#x6$j2Vq$bmTRauFe`Fz5+NHCdGe!ktgN30rK(_}v?x`F<)QZQ6x%45 zWmL*CuEZoM6U~Og{H(3YWM-u>gH=yh1w(GmQPWsr+8^@b?Ys`K|DgwoflN?bDZh z2OxEy#BSGnJSyze8;^{g`s0U>M*0Zzp*eh7m^P|EGPB{ptRuEZe0;;bP=+~Mqp+{IFtP^OJ+4zSHt7*#&ih unbind dataset +# +def bindDataWithProject(userId, projectId, version, dataIds, isUbind = False): + if not isUbind: + #bind dataset + paths = [] + for i in range(len(dataIds)): + pathPj = config.dir_home + config.dir_home_user + '/' + str(userId) + '/system/' + str(projectId) + '/' + str(version) + '/dataset/' + if not os.path.exists(pathPj): + os.mkdir(pathPj) + pathDset = config.dir_home + config.path_dataset + '/' + str(dataIds[i]) + if not os.path.exists(pathDset): + return { + 'status' : 0, + 'result' : 'Dataset file not exists!' + } + shell.execute('ln -s ' + pathDset + ' ' + pathPj) + pathDsetln = 'dataset/' + str(dataIds[i]) + '/' + paths.append(pathDsetln) + + return { + 'status': 1, + 'result': { + 'message':"dataset bind successde!", + 'path': paths + } + } + + else: + #unbind dataset + done = {} + for i in range(len(dataIds)): + path = config.dir_home + config.dir_home_user + '/' + str(userId) + '/system/' + str(projectId) + '/' + str(version) + '/dataset/' + str(dataIds[i]) + if not os.path.exists(path): + #the data not binded + done[str(i)] = 'data hava not been binded before!' + elif os.path.exists(path): + try: + shell.execute('rm -rf ' + path) + done[str(i)] = 1 + except Exception as e: + sysout.err(TAG, e) + done[str(i)] = str(e) + success = True + for k in range(len(dataIds)): + if not done[str(k)] == 1: + success = False + + if success : + return { + 'status': 1, + 'result': "unbind datasets success!" + } + else: + return { + 'status': 0, + 'result': done + } # # -# cmd: +# cmd:path_dataset # # def generateCode(userId, projectId, projectName, version): # (exist, result) = checkVersion(userId, projectId, version) diff --git a/project/main/manager/vmManager.py b/project/main/manager/vmManager.py index 8d6f99f..c6fca0c 100644 --- a/project/main/manager/vmManager.py +++ b/project/main/manager/vmManager.py @@ -8,7 +8,7 @@ TAG = "vmManager" -def startVm(userId, isoName, vmId, passwd, gpu, cpu, memory, isoRemarks, action='start'): +def startVm(userId, isoName, vmId, passwd, gpu, cpu, memory, isoRemarks, action='start', pstartTime = None, pendTime = None): sysout.log(TAG, 'startVm...') host = config.vms_host port = config.vms_port @@ -24,6 +24,12 @@ def startVm(userId, isoName, vmId, passwd, gpu, cpu, memory, isoRemarks, action= "cpu": cpu, "mem": memory } + + if not pstartTime == None: + #package device + p_body["begintime"] = pstartTime + p_body["endtime"] = pendTime + p_body = json.dumps(p_body) params = ("new_task_publish@" + p_body).encode('utf-8') # params = json.dumps().encode('utf-8') diff --git a/project/main/server/__pycache__/httpServer.cpython-36.pyc b/project/main/server/__pycache__/httpServer.cpython-36.pyc index 6b6b8ce91037ca8e3b5764f09f0aa392bb07f2dc..91d9ddbec0d524047e417462bbcf62e82dc3fdd9 100644 GIT binary patch delta 1321 zcmZ`&-D@0G6u)hVy z-u;OilgH+xC^%>R_U6lnh4@1}^b`=Eg(bH?JN9ifu8!OgYop3q7PX1*?%lB~Rn#V6 zuW-+-G+#TX0wuZ;ti1D)>DnXaZ|BnTekxQnCc2?2r=sQ3ZQwF+q$)YS16% zb|dpUjaYGH{&8ny)%;aBY|eVqCukaLr5U(PHfV@J?hbKu9Bv(!T!ItQmbUr8J9)_R z<=_pisyremB^@`vcsJb|+Gy6a`~^8@_Whdt#N74A_nt+1L-H7SJQz=rlXMFKb|7IE zb}g~%0%BWPT?e=@CY0T>Xhu0*XXj~TTy#i$J$w)T0QQ0HhxmFQe7XPewbv)iJs`}z z?yid1S{13SM9WG()`isWNmayDpQ}KXRQLr{)%KQ?%G63xO`V>@WP{!4H5$RPFJx|g zLpR%T7QGk0-Hg?%bzM&ydWA};QV5rM3b~~{l@RZqe7N0mW4ohwj&cRvPt`0caxH+X zxzuNn6MUGm!qrAhHb;*Srnu77o3B%QtFfVL9zDjb2NUKCqvw!Y^P0(J{)OhmTWEPi z*#)$G2}>s7a%CYY^tD2Gi{*vLp=5D~WMC2ZA1G)6IIzBg#wzySO{MZd-lw{hik@5+ zz)a@N0e(R0fWlK4VAtz2_yP?B5w2VR%+V7Z@u^uW4Ym!gXx{u>T-Xa~l*`Ny@r}&v zX8cY&E3BNmNVA;$zrpfdfbX0$p8pY6vSNpz3G4U+s>u|b00%a>81e>3-fRb@(%=GE zwD|qC;469Nc^>WzT&zmnI3670(wOz-_STvWzd0dwd z!)N5L=Ev}aJa7ICcT4Aa#24Akn(gTH@O^}d3nw5_Rniio+L-w%nvgG<-=pa_P9u?p zakAcNlV0D{x&iYQuQ)R$YzGV;iDYnW2Gl7FI8bui6cOnlepY;JIiJNyivO##?O~Lr@W)j5@y{l!C{fh?#=x zR*_!kl!r?}Q_->+o{0%_un-tQAt=7aIRk9YGp|Xly`FaBsIMRGSK%aPz-a<%1jY!e z!nEqkCJfWTAt2pkz7uW4GPl-PfeMlTO~rtxz%h!?iKo`6_5$5xOzc|YIR*Upht?Z@ zxkllObk)pC$vlMZK;}A83L{8MwFl(W*fM`7i6g0Y<93jMFZN14YVfl(%lF0ca+NQN zX1Q-JQpE*=ia01Q)IOt|Xb4OpDS<@shHnl%NN#r^+?Y)|_j=v1m4t9V1R&|)lBhY; zYxC$zD@t0u7}C~en)aYUl#bM9oTRWscb5q=W&VngNtcO8-vT~C)>cz|aL)6y;+r$B KSB$dZ8txx+dy-ZF diff --git a/project/main/server/httpServer.py b/project/main/server/httpServer.py index 3278729..6edf845 100644 --- a/project/main/server/httpServer.py +++ b/project/main/server/httpServer.py @@ -105,6 +105,8 @@ def runWithVm(request_body): cpu = None memory = None action1 = None + pstartTime = None + pendTime = None try: userId = request_body['userId'] pjId = request_body['projectId'] @@ -117,12 +119,18 @@ def runWithVm(request_body): gpu = request_body['gpu'] cpu = request_body['cpu'] memory = request_body['memory'] + if 'action1' in request_body: action1 = request_body['action1'] + if 'pstartTime' in request_body: + pstartTime = request_body['pstartTime'] + if 'pendTime' in request_body: + pendTime = request_body['pendTime'] + except Exception as e: return str(resp_err_params) + str(e) return projectManager.runWithVm(str(userId), str(pjId), pjName, str(version), str(vmId), passwd, isoName, - isoRemarks, gpu, cpu, str(memory), action1) + isoRemarks, gpu, cpu, str(memory), action1, pstartTime, pendTime) ## @@ -147,6 +155,30 @@ def getMyFiles(request_body): } +def bindDataWithProject(request_body): + userId = None + projectId = None + version = request_body['version'] + dataIds = None + isUbind = False + + if request_body['action'] == 'unbindDataset': + isUbind = True + try: + userId = request_body['userId'] + projectId = request_body['projectId'] + # pjName = request_body['projectName'] + version = request_body['version'] + dataIds = request_body['dataIds'] + except Exception as e: + return str(resp_err_params) + str(e) + if dataIds == None or len(dataIds) == 0: + return {'status': '0', 'result': 'empty dataset!'} + else: + return projectManager.bindDataWithProject(userId, projectId, version, dataIds, isUbind) + + + def praseData(request_body): if type(request_body) != type({}): return resp_err_params @@ -169,6 +201,10 @@ def praseData(request_body): return deleteProject(request_body) elif action == 'getMyFiles': return getMyFiles(request_body) + elif action == 'bindDataset': + return bindDataWithProject(request_body) + elif action == 'unbindDataset': + return bindDataWithProject(request_body) else: return {'status': 0, 'result': 'request & params not support!!!'} diff --git a/project/test/codeTest.py b/project/test/codeTest.py index c8c6c12..28e5486 100644 --- a/project/test/codeTest.py +++ b/project/test/codeTest.py @@ -1,3 +1,22 @@ # a = {'action':1} # print('action' in a) +# p_body = { +# "iname": 1, +# "cname": 2, +# "user": 3, +# "action": 4, +# "password": 2, +# "nname": 2, +# "gpu": 2, +# "cpu": 2, +# "mem": 2 +# } +# +# print(len(p_body)) +# #package device +# p_body["begintime"] = 11 +# p_body["endtime"] = 11 +# +# print(p_body) +# print(len(p_body)) \ No newline at end of file From a5e929ccf9143eaeb3617d05a684d4bbf03b31d8 Mon Sep 17 00:00:00 2001 From: JerryYin Date: Wed, 31 Oct 2018 16:53:10 +0800 Subject: [PATCH 02/42] changed dataset dir from /base/file to /user/system/file --- .idea/workspace.xml | 74 ++++++++++++-------------- project/main/manager/projectManager.py | 10 +++- 2 files changed, 42 insertions(+), 42 deletions(-) diff --git a/.idea/workspace.xml b/.idea/workspace.xml index a4b7e33..e8f169c 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -2,13 +2,7 @@ - - - - - - - type( - isconn type({}) send Server response form error @@ -212,8 +204,10 @@ getAllFiles system .system - path_dataset shell + bindDataWithProject + path_dataset + pathDset @@ -244,8 +238,8 @@ @@ -527,7 +521,7 @@ - + @@ -879,10 +873,10 @@ - + - - + + @@ -891,8 +885,8 @@ - - + + @@ -913,31 +907,31 @@ - + - - + + - + + + - - + + - + - - + + - - - + diff --git a/project/main/manager/projectManager.py b/project/main/manager/projectManager.py index 4a2be28..55f415d 100644 --- a/project/main/manager/projectManager.py +++ b/project/main/manager/projectManager.py @@ -42,6 +42,11 @@ def createPreProject(userId, projectId, projectName, projectType): os.makedirs(pjHome) shell.execute('cp ' + config.file_system_readme + ' ' + pjHome + '/') + #create directory for users's dataSets + pathDsets = config.dir_home + "/" + config.dir_home_user + '/' + str(userId) + '/system/datasets' + if not os.path.exists(pathDsets): + os.makedirs(pathDsets) + path = config.dir_home_user + '/' + str(userId) + '/system/' + str( projectId) + '/1' # 1--version of pj， vesionInit=1 dir = config.dir_home + path @@ -156,7 +161,8 @@ def bindDataWithProject(userId, projectId, version, dataIds, isUbind = False): pathPj = config.dir_home + config.dir_home_user + '/' + str(userId) + '/system/' + str(projectId) + '/' + str(version) + '/dataset/' if not os.path.exists(pathPj): os.mkdir(pathPj) - pathDset = config.dir_home + config.path_dataset + '/' + str(dataIds[i]) + # pathDset = config.dir_home + config.path_dataset + '/' + str(dataIds[i]) + pathDset = config.dir_home + "/" + config.dir_home_user + '/'+str(userId) + '/system/datasets/' + str(dataIds[i]) if not os.path.exists(pathDset): return { 'status' : 0, @@ -181,7 +187,7 @@ def bindDataWithProject(userId, projectId, version, dataIds, isUbind = False): path = config.dir_home + config.dir_home_user + '/' + str(userId) + '/system/' + str(projectId) + '/' + str(version) + '/dataset/' + str(dataIds[i]) if not os.path.exists(path): #the data not binded - done[str(i)] = 'data hava not been binded before!' + done[str(i)] = 'data hava not been binded bebindDataWithProjectfore!' elif os.path.exists(path): try: shell.execute('rm -rf ' + path) From f54af51bd630e2aa33ad2050e5dfcab7245272a2 Mon Sep 17 00:00:00 2001 From: JerryYin Date: Thu, 1 Nov 2018 14:31:55 +0800 Subject: [PATCH 03/42] done delete dataset --- .idea/workspace.xml | 214 ++++++++++-------- .../config/__pycache__/config.cpython-36.pyc | Bin 1691 -> 1655 bytes project/main/config/config.py | 2 +- .../__pycache__/fileManager.cpython-36.pyc | Bin 2564 -> 2565 bytes .../__pycache__/jupyter.cpython-36.pyc | Bin 1221 -> 1211 bytes .../__pycache__/projectManager.cpython-36.pyc | Bin 4629 -> 5837 bytes project/main/manager/fileManager.py | 2 +- project/main/manager/jupyter.py | 2 +- project/main/manager/projectManager.py | 76 ++++++- .../__pycache__/httpServer.cpython-36.pyc | Bin 4356 -> 4704 bytes project/main/server/httpServer.py | 19 +- 11 files changed, 208 insertions(+), 107 deletions(-) diff --git a/.idea/workspace.xml b/.idea/workspace.xml index e8f169c..92dc93e 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -2,7 +2,12 @@ + + + + + - - + + - + - + - + - - + + @@ -54,7 +59,7 @@ - + @@ -62,27 +67,18 @@ - + - + - - - - - - - - - - - + + @@ -99,7 +95,7 @@ - + @@ -111,34 +107,45 @@ - - + + - - + + + - + - - + + + + + + + + + + + + - - + + @@ -147,10 +154,10 @@ - + - - + + @@ -158,8 +165,8 @@ - - + + @@ -178,8 +185,6 @@ - type({}) - send Server response form error sendSocket projectId @@ -194,7 +199,6 @@ jupyter.executingNb executingNb ns_doname - dir_home_user file_system_readme projectName resp_err_params @@ -206,8 +210,11 @@ .system shell bindDataWithProject - path_dataset pathDset + startVm + sys + path_dataset + dir_home_user @@ -233,13 +240,13 @@ @@ -521,15 +528,15 @@ - + - + - + @@ -577,10 +584,18 @@ + + + + + file://$PROJECT_DIR$/project/main/server/httpServer.py + 187 + + + + - - - @@ -745,13 +760,6 @@ - - - - - - - @@ -814,13 +822,6 @@ - - - - - - - @@ -856,26 +857,23 @@ - + - - + + - + - - - - - + + - + @@ -883,58 +881,84 @@ + + + + + + + + + + - - + + - + - - + + - + - - + + - + - - - - - - - + + - + + + + + + + + - - + + + + + + + + - - + + + + + + + + + + \ No newline at end of file diff --git a/project/main/config/__pycache__/config.cpython-36.pyc b/project/main/config/__pycache__/config.cpython-36.pyc index 607aa0c88a9bfe2d1b73b2288ec3e984022f1db7..e1eefb30725752267bc601487cf769066dd4ea0e 100644 GIT binary patch delta 167 zcmbQu`<+MGn3tC;sQOm)L0$%i#|%h-8OU}3;^G|>mHpXLxl{PW877uEsilagNVG6S zNu)@oNC9cd6zLQhAT5<5n<59KrBmdC88j6pKH0N*1LIU?Mw!WOtScw~Wlga#0UE}_ l2!sMGOh8f{%w`l}2J)CVxLEi(7=e%piDc#w<6!1s1^|vf9T5Nk delta 159 zcmey)Gn-f0n3tF9^oQ%w`*|4{9y1^TW+2-Eh>Q13RQ6{|;hR|Dq?gK_A}-DlC6OYL zBH6+aC7B|XA`PUaQe;wOfwXjrT#7u9mPt_vX3$ie_+Sq|pMF|qPO5%NVo73gYDw|t r*^JYe8D%FMv99Ec;weZh$%uzaO}@>V!e}X%*o#=H=zeD83b4wUM)egPpx7S3$QZZF4(EEh7MoObI9e delta 30 lcmZn_X%XQx=H=!3a^qTb`UcRwSnulw{`TZQjf*#0UV#91C;+ delta 44 zcmdnZd6bjWn3tE!|MT_eeH%HaF^dTM=BJeAr26HTxaF7TrMMOqwSOJ18L=;1a6$G(}jIiT`NM?I6UYo`484|PV zVGoNU1bwapDju;*>v-0}{%8uerp3Ac{DpYG&80VVhel30Z|mAxH2U6 zihZ~;ER0!JKKSeNWuA3V&&2To>5oF^q82x~;6)a(8WdT=X^ORoH;XJbBE4qROl7nn z`~`W7I7pHFc{N{IEog(xA)ir}0>zc4%$5<`QE;V}jdH?4 zb(_>%9GJ3anQ#>v#e;h@T_!y|Z^N1hAB0i$ZbPfAm0m)rk?So?h{b3ztVUtyef>E& z9oHE57M!)k=+32N^{jCY#yac99ZCfmsAH0ycW=y@CGkwG48ZvEhiJN3kgyDJ%q61sA460+E?UnBELXm%KW zo!xEHMzKlf7%&-`HLGS*1jE0PcZkCbNLMjrc>h3{!V>mlKFcDv<_Sk(4hXl%D1;;} zyg{3^VeBXa^TMIO>-*-I@I^)$B6!T3si9>SyXexN|Krkj;!+P?>i1oWiS_bLPThI; z!2`wKfPRyz;Z4rV;tl-zo&V$Pof~d*{A=T%Y}IqOhwkR#Ly=Q1nloE)p4HXBKajNt z<4Fke%1d3}oB6#!_^&kQSba)oQD(($bNLUjI0LLP7BvGZNR^>N#7#0S7*zR`34G(s z!^}LZ^Lfnj^%_R65C4AkHRw*%s`@=&ta#79vu`L0eld%rS*uc@d*Nl!jZF z%1yaRK~+CIBS*2$(!#g49Lr9RQf_ZPQ{|6o=+c3+_LZT9GUro3A^Axqb%8>9E@ zx0dUo)E6#=_bZiZ7#=R?yM|n>FIBtRYPBv2TxFI>&@pp|I;V2uyqxP)bJwi}gAnDN z&TqNX&BrJa-a^8_V?S)T(BgdLO1X92-0uRn(IdB{rp?HEa^Mt>< zU^-@5OzXS-L!jwnc&ydWkW`YP{eG~eD>z~lil%L1n;NwXg}4>#8yk@sSxQgA@aCP^ zP`WZO6>!96k4M}>c(nT4Dr?1*PjH41Q?muxvC#&CjIdX@e4=dOpoXKGws4VgBS$gi zC|AJ}2(M)zmOPnbcnFtJ_;Vh6v9G)&`#_!|j3I=<=8{!@%3%>qVi9;)1R%fc6~Vs9 zuVtbjtpSn6a?2oqq?$?YH`C-#GBv+@rtV13iLP;bd9@mLwY9HA!|o;G#HNiA-K2K2 zNloUuJdU(c6xG#gRPCqht8C?xT6j~sNF_#}HL^X?+ly#j94?i!gGo$c)U+fSU+|Wy z(S_C3X*!4eh?XxS=^EkM>T;AMnxxJquM(k@lOGcqLsAYTHNLJ%2X9^$azD-%{C<-q zlU=KFuO3zFk^F?T=7`Lbl#VwJWr5~pBwf97Phh>>4VUCBEe>yy5&E4;Z*(bb>VM%< zIEjQMMW$;m9GHS*SXqaVEnuzrSUUmu+y)y)v;b=-tsI|YMIkR2QMs(ickwebQ-E_lXscd`sd6DK_ z%(u}FYutAdA2*^VaN0wpp-2mQ2)7d-65r_|ktIouDhoYZW^10ZTOhumms7R^MU|nf zmK8bDujOTdyI9B3*cJt%oJqnn3M=CMY@$s>7O!nMqv5SkRUyrXa-{*`?JGx2lpMDw0zV)y{^MEc=b9~e^!)H7K)$Ir%R=2!WnO7mBqa$v{Pu2L~Qx%CT<`9hug-Bqip)o z*%quoc#`K7!*lF~17vBf`R`Dy-5N-k_$r$+VE z@^s5!QLhCX5oZX;U*z5PP9s{|G?4+%Cn^Vu;k!wAA<8Wu*~EEY|6q8GGb1PJgp~zhwb-Afn>glfTp1__iVN<(lQ;#+> zqkLA*hE*9qEr__kI6UQuYvo-B2+V0mi1kqvLo&sW2648z9OoAtTHIxnZd zu$&$aR=V1jn5{2V+s4&uUFeAyzFzXNDh~K<@5ZdWeDUvnkBd7UF^b2PeEhyR6%Pk{ zn~yLIPvL?SMNJ4`UuUAgqZC}6uWs-SioR#-(?j4>A2TTUP{7!O5z7S^r!l})?=Y%k zTcb@1LYJ->Vn4nnCMqyq+0r)WkiEc3Bf^OLOXqAokm6L_DviKE{G!z7^tnT9x6A$j D&npVn diff --git a/project/main/manager/fileManager.py b/project/main/manager/fileManager.py index a20d7d8..e51767a 100644 --- a/project/main/manager/fileManager.py +++ b/project/main/manager/fileManager.py @@ -130,7 +130,7 @@ def createProject(path, projectId, projectName, type, filePath): def deleteFile(path): try: if os.path.exists(path): - d = shell.execute('rm -r ' + path) + d = shell.execute('rm -rf ' + path) print("d ======== ") print(d) return (1, 'Delete Sccess!') diff --git a/project/main/manager/jupyter.py b/project/main/manager/jupyter.py index 2af535b..36733ee 100644 --- a/project/main/manager/jupyter.py +++ b/project/main/manager/jupyter.py @@ -33,7 +33,7 @@ def executingNb(path, nbName, kernel): nbook = nbformat.read(f, as_version=4) ep = ExecutePreprocessor(timeout=600, kernel_name=kernel) (nbd, res) = ep.preprocess(nbook, {'metadata': {'path': runPath}}) - except ModuleNotFoundError as e: + except Exception as e: sysout.err(TAG, e) diff --git a/project/main/manager/projectManager.py b/project/main/manager/projectManager.py index 55f415d..99fee5e 100644 --- a/project/main/manager/projectManager.py +++ b/project/main/manager/projectManager.py @@ -158,11 +158,19 @@ def bindDataWithProject(userId, projectId, version, dataIds, isUbind = False): #bind dataset paths = [] for i in range(len(dataIds)): - pathPj = config.dir_home + config.dir_home_user + '/' + str(userId) + '/system/' + str(projectId) + '/' + str(version) + '/dataset/' + pathPj = "" + pathDset = "" + if config.dir_home_user != '': + pathPj = config.dir_home + config.dir_home_user + '/' + str(userId) + '/system/' + str(projectId) + '/' + str(version) + '/dataset/' + pathDset = config.dir_home + config.dir_home_user + '/' + str(userId) + '/system/datasets/' + str(dataIds[i]) + else: + pathPj = config.dir_home + '/' + str(userId) + '/system/' + str(projectId) + '/' + str(version) + '/dataset/' + pathDset = config.dir_home + '/' + str(userId) + '/system/datasets/' + str(dataIds[i]) + if not os.path.exists(pathPj): os.mkdir(pathPj) # pathDset = config.dir_home + config.path_dataset + '/' + str(dataIds[i]) - pathDset = config.dir_home + "/" + config.dir_home_user + '/'+str(userId) + '/system/datasets/' + str(dataIds[i]) + # pathDset = config.dir_home + config.dir_home_user + '/'+str(userId) + '/system/datasets/' + str(dataIds[i]) if not os.path.exists(pathDset): return { 'status' : 0, @@ -184,13 +192,22 @@ def bindDataWithProject(userId, projectId, version, dataIds, isUbind = False): #unbind dataset done = {} for i in range(len(dataIds)): - path = config.dir_home + config.dir_home_user + '/' + str(userId) + '/system/' + str(projectId) + '/' + str(version) + '/dataset/' + str(dataIds[i]) + path = '' + path1 = '' + if config.dir_home_user != '': + path = config.dir_home + config.dir_home_user + '/' + str(userId) + '/system/' + str(projectId) + '/' + str(version) + '/dataset/' + str(dataIds[i]) + path1 = config.dir_home + config.dir_home_user + '/' + str(userId) + '/system/' + str(projectId) + '/' + str(version) + '/dataset' + else: + path = config.dir_home + '/' + str(userId) + '/system/' + str(projectId) + '/' + str(version) + '/dataset/' + str(dataIds[i]) + path1 = config.dir_home + '/' + str(userId) + '/system/' + str(projectId) + '/' + str(version) + '/dataset' + if not os.path.exists(path): #the data not binded - done[str(i)] = 'data hava not been binded bebindDataWithProjectfore!' + done[str(i)] = 'data hava not been binded before!' elif os.path.exists(path): try: - shell.execute('rm -rf ' + path) + shell.execute('rm -r ' + path) + shell.execute('rm -r ' + path1) done[str(i)] = 1 except Exception as e: sysout.err(TAG, e) @@ -210,6 +227,55 @@ def bindDataWithProject(userId, projectId, version, dataIds, isUbind = False): 'status': 0, 'result': done } + + +def deleteDataset(userId, dataId): + try: + path = '' + pathDsetFile = '' + + #first unbind dataset of all project + if config.dir_home_user != '': + path = config.dir_home + config.dir_home_user + '/' + str(userId) + '/system/' + pathDsetFile = config.dir_home + config.dir_home_user + '/' + str(userId) + '/system/datasets/' + str(dataId) + else: + path = config.dir_home + '/' + str(userId) + '/system/' + pathDsetFile = config.dir_home + '/' + str(userId) + '/system/datasets/' + str(dataId) + + + content = fileManager.getAllFiles(path) + for i in range(len(content)): + # is project's dataset dir ? == 1. 'dataset/dataId' is in pwd and 2. len(pwds) == 8 + files = (content[i])['files'] + pwd = (content[i])['pwd'] + if ('dataset/' + str(dataId)) in pwd: + pwds = str(pwd).split('/') + if len(pwds) == 8: + # pwd = '/notebook/storage/userId/system/projectId/version/dataset/dataId' + projectId = pwds[4] + version = pwds[5] + bindDataWithProject(userId, projectId, version, [dataId], True) + + #second delete file + (code, msg) = fileManager.deleteFile(pathDsetFile) + if code == 1: + return { + 'status': 1, + 'result': "delete dataset success!" + } + else: + return { + 'status': 0, + 'result': msg + } + except Exception as e: + sysout.err(TAG, e) + return { + 'status': 0, + 'result': "delete failed! " + e + } + + # # # cmd:path_dataset diff --git a/project/main/server/__pycache__/httpServer.cpython-36.pyc b/project/main/server/__pycache__/httpServer.cpython-36.pyc index 91d9ddbec0d524047e417462bbcf62e82dc3fdd9..cb136a5e11b6794ba899eac07c719f6b6b4a0a4b 100644 GIT binary patch delta 1127 zcmZ`%OK%iM5bo}I?06o#3mAwob_9mN;sqq^@DK*YK!gdFFMxs!!dmD77MoqGrUk+p z#|LxBAyO2zloJw=9JwIifH-jAgp@;WM4B9e1pk5yUyXU>kj$vQud2Gbr|Rpuz4`T) zp^+pB$A2x(?bs;98}YBlf&UFGd-luFqhvr2-WJn+TAdP&FYg!Ll3I+lSWIBmTR-JO zJAGnNf?cozdte9bgI#b1?12NY5AFe1z#%vQN8lcC3=Y8wIMUTz%=+{j;ws`;*UI=& zE>`Or5{a&ti5DoS+gbm5b=4I0qv^uSOpA>D$A4gdt~C3o!AY4?FL(vkve&eKR?f(# z-5T8UcW`3Z{t@hzJFV*ZR_=K`(X(BC%-;wP4}Fc2=}J30bFtmgwDZ$h<*DU?`|Mxg z?xlTLxg-JnxUf~Qqp)la0L>&!-Rec#5@WD#3aq{tou(^{mP8y1ERD>a zWw~CQ;@!><&RNk>XGN~Gm{oLMJKELW9bArauDN-?%+)dijq-*+MG-z)__}3UC$yam zH!pd#GA!~`XW$--M5lA9Nspl}OHIQi?z-Ge``P=ic@JRW2`pm?gmm#L8OT`H^IC4^#GEouU!!TBx)xnRd5xh}I#Z z4Gc|&L54km>4s;3_f97C1qbDz{8txlyK}i)IH$*t(=f06--*QcSPsxo zD`Je;ALD`bFObeyL4h${706yMz_9jdJRrv&zl!h5gNJzMVL&4)zfri9wshgNJBIp@ zF1&eSpHC^=YgxBbcyryB&S=a&td8Ihd|TZmU)wdc-EzVntt~_oeBwC6uzgwEKl%=C z<^zOcwP_MwW5e{?o7t5%rI&YQ?d$V%=}eZ=jg*Lmrz3W>zU{(6xJBH_X6CzOX094D zPv3E>g4ES+X6OXho@6K``>akWbtyXQ{aGCvsu=8f@&}r^uGsU CX6Cp6 delta 724 zcmYjOOKTHR6ux&duguJy&a|;HhO}ZzG!Lw8YG^FP2gce>H&UW@Q6e{x7)ipN*(eQe z>_SjM4uT88*HUnifm(tay> zivF$dS&jS(w%jy&!Pc$bfQ!oEtaxfXum(S^MOuS~lcDe7y7P91TioW(3rvd>OpI-c zWtd~>Tv^_pldvSG+H&OOcGwO&p{PQO)u;ncShie}Ewd8vci_en4zGzC+M*(SSzU%N zEI0E3Wl%sM)T63m(v$(dn}*}AnovZ2zwExGYv*MD8bZ+)@&%D9I&B^$dV^48q}N75 zYMDu--VTO?NV_v^^FY)=^Qst7-8)S`LDyTPO?d9zvzyo{8WKv7^H*mNPzE{zQ5;-E zY@jM;pzr7Cd3flbys?Uf$QcAXqoD|Qx?wmH>#}5EsXGipaRr^YD#4TRFM{~%;~Zpi aaRj0+RlLInxYGq%fV0yJ+EH`f%>D-XubEB& diff --git a/project/main/server/httpServer.py b/project/main/server/httpServer.py index 6edf845..4ae4107 100644 --- a/project/main/server/httpServer.py +++ b/project/main/server/httpServer.py @@ -104,7 +104,7 @@ def runWithVm(request_body): gpu = None cpu = None memory = None - action1 = None + action1 = 'start' #default : start pstartTime = None pendTime = None try: @@ -177,6 +177,18 @@ def bindDataWithProject(request_body): else: return projectManager.bindDataWithProject(userId, projectId, version, dataIds, isUbind) +def deleteDataset(request_body): + userId = None + dataId = None + try: + userId = request_body['userId'] + dataId = request_body['dataId'] + except Exception as e: + return str(resp_err_params) + str(e) + if dataId == None: + return {'status': '0', 'result': 'empty dataset!'} + else: + return projectManager.deleteDataset(userId, dataId) def praseData(request_body): @@ -185,12 +197,10 @@ def praseData(request_body): else: action = request_body['action'] sysout.info(TAG, 'action=' + str(action)) - # functions = { # 'initproject': initProject(request_body), # 'newVersion': newVersion(request_body) # } - if action == 'initProject': return initProject(request_body) elif action == 'newVersion': @@ -205,10 +215,11 @@ def praseData(request_body): return bindDataWithProject(request_body) elif action == 'unbindDataset': return bindDataWithProject(request_body) + elif action == 'deleteDataset': + return deleteDataset(request_body) else: return {'status': 0, 'result': 'request & params not support!!!'} - # server def application(environ, start_response): # 定义请求的类型和当前请求成功的code From f894855929457a16c0c15124440de2ebcee3855e Mon Sep 17 00:00:00 2001 From: JerryYin Date: Mon, 5 Nov 2018 19:17:27 +0800 Subject: [PATCH 04/42] fixed bugs of new version --- .idea/workspace.xml | 144 ++++++++++++------------- project/main/config/config.py | 5 +- project/main/manager/projectManager.py | 53 +++++---- 3 files changed, 105 insertions(+), 97 deletions(-) diff --git a/.idea/workspace.xml b/.idea/workspace.xml index 92dc93e..21324d6 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -4,10 +4,7 @@ - - - - + - - + + @@ -107,8 +105,8 @@ - - + + @@ -119,14 +117,13 @@ - + - - + + - @@ -135,8 +132,8 @@ - - + + @@ -144,7 +141,7 @@ - + @@ -156,8 +153,8 @@ - - + + @@ -165,8 +162,8 @@ - - + + @@ -185,12 +182,6 @@ - Server response form error - sendSocket - projectId - dir_home_user = '/users' - deleteProject - base getNotebookName ns_host request & params not support!!! @@ -204,17 +195,23 @@ resp_err_params runWithVm config - dir_home getAllFiles system .system shell bindDataWithProject - pathDset startVm sys path_dataset + ln -s + config.dir_home + config.dir_home_user + '/' + str(userId) + pathDset + path1 dir_home_user + getDirNumber + resp_err_version_invlid + dir_home + custom.css @@ -245,8 +242,8 @@ @@ -528,7 +525,7 @@ - + @@ -536,7 +533,7 @@ - + @@ -864,6 +861,20 @@ + + + + + + + + + + + + + + @@ -881,19 +892,22 @@ - + - - + + - + + + + - + @@ -901,64 +915,46 @@ - + - - + + - + - - + + + + + - + - - + + - - - - - - - - - - - - - - - + + - - + + - - - - - - - - \ No newline at end of file diff --git a/project/main/config/config.py b/project/main/config/config.py index 73a5e95..bd57f05 100644 --- a/project/main/config/config.py +++ b/project/main/config/config.py @@ -19,9 +19,9 @@ ## http request server for vm vms_port = 18883 -# vms_host_dev = '121.40.62.80' #dev +vms_host_dev = '121.40.62.80' #dev vms_host_release = '60.12.136.59' #release -vms_host = vms_host_release +vms_host = vms_host_dev # ----------- RELEASE ------------ @@ -42,7 +42,6 @@ # file_system_readme = dir_home +'/base/readme' - # ----------- DEV ------------ ns_host = '172.16.3.254' ns_port_http = 8100 diff --git a/project/main/manager/projectManager.py b/project/main/manager/projectManager.py index 99fee5e..b637d4e 100644 --- a/project/main/manager/projectManager.py +++ b/project/main/manager/projectManager.py @@ -21,8 +21,7 @@ def checkVersion(userId, projectId, v): if v == None: return (False, resp_err_version_invlid) - maxVersion = int(fileManager.getDirNumber( - config.dir_home + config.dir_home_user + '/' + str(userId) + '/system/' + str(projectId) + '/')) + maxVersion = int(fileManager.getDirNumber(config.dir_home + config.dir_home_user + '/' + str(userId) + '/system/' + str(projectId) + '/')) if (v <= 0 or v > maxVersion): return (False, resp_err_version_invlid) return (True, maxVersion) @@ -81,16 +80,18 @@ def createNewVersion(userId, projectId, projectName, versionCur): if not os.path.exists(config.dir_home + curPath): return resp_err_version_old_err else: + # cmd = 'cp / notebook / storage / 416 / system / 35 / 15 / edit.ipynb / notebook / storage / storage / 416 / system / 35 / 26 /' shell.execute('cp ' + curNb + ' ' + dir + '/') shell.execute('cp ' + curH5 + ' ' + dir + '/') shell.execute('cp ' + curPY + ' ' + dir + '/') + shell.execute('cp ~/.jupyter/custom/custom.css ' + dir + '/') return { 'projectId': projectId, 'projectName': projectName, 'version': version, - 'notebook': config.ns_doname + '/notebooks' + path + '/' + config.getNotebookName(), - 'html': config.ns_doname + '/notebooks' + path + '/' + config.getH5Name(), - 'py': config.ns_doname + '/notebooks' + path + '/' + config.getPYName() + 'notebook': config.ns_doname + '/notebooks/storage' + path + '/' + config.getNotebookName(), + 'html': config.ns_doname + '/notebooks/storage' + path + '/' + config.getH5Name(), + 'py': config.ns_doname + '/notebooks/storage' + path + '/' + config.getPYName() } else: return resp_err_version_create_dir_err @@ -162,20 +163,24 @@ def bindDataWithProject(userId, projectId, version, dataIds, isUbind = False): pathDset = "" if config.dir_home_user != '': pathPj = config.dir_home + config.dir_home_user + '/' + str(userId) + '/system/' + str(projectId) + '/' + str(version) + '/dataset/' - pathDset = config.dir_home + config.dir_home_user + '/' + str(userId) + '/system/datasets/' + str(dataIds[i]) + # pathDset = config.dir_home + config.dir_home_user + '/' + str(userId) + '/system/datasets/' + str(dataIds[i]) else: pathPj = config.dir_home + '/' + str(userId) + '/system/' + str(projectId) + '/' + str(version) + '/dataset/' - pathDset = config.dir_home + '/' + str(userId) + '/system/datasets/' + str(dataIds[i]) + # pathDset = config.dir_home + '/' + str(userId) + '/system/datasets/' + str(dataIds[i]) + + # pathPj = '/data/system/' + str(projectId) + '/' + str(version) + '/dataset/' + pathDset = '/data/system/datasets/' + str(dataIds[i]) if not os.path.exists(pathPj): os.mkdir(pathPj) # pathDset = config.dir_home + config.path_dataset + '/' + str(dataIds[i]) # pathDset = config.dir_home + config.dir_home_user + '/'+str(userId) + '/system/datasets/' + str(dataIds[i]) if not os.path.exists(pathDset): - return { - 'status' : 0, - 'result' : 'Dataset file not exists!' - } + shell.execute('mkdir '+ pathDset) + # return { + # 'status' : 0, + # 'result' : 'Dataset file not exists!' + # } shell.execute('ln -s ' + pathDset + ' ' + pathPj) pathDsetln = 'dataset/' + str(dataIds[i]) + '/' paths.append(pathDsetln) @@ -201,17 +206,25 @@ def bindDataWithProject(userId, projectId, version, dataIds, isUbind = False): path = config.dir_home + '/' + str(userId) + '/system/' + str(projectId) + '/' + str(version) + '/dataset/' + str(dataIds[i]) path1 = config.dir_home + '/' + str(userId) + '/system/' + str(projectId) + '/' + str(version) + '/dataset' + # path = config.dir_home + config.dir_home_user + '/' + str(userId) + '/system/' + str(projectId) + '/' + str(version) + '/dataset/' + str(dataIds[i]) + + haveException = False + try: + shell.execute('rm -r ' + path) + # shell.execute('rm -r ' + path1) + except Exception as e: + sysout.err(TAG, e) + done[str(i)] = str(e) + havaException = True + if not os.path.exists(path): #the data not binded - done[str(i)] = 'data hava not been binded before!' - elif os.path.exists(path): - try: - shell.execute('rm -r ' + path) - shell.execute('rm -r ' + path1) - done[str(i)] = 1 - except Exception as e: - sysout.err(TAG, e) - done[str(i)] = str(e) + # done[str(i)] = 'data hava not been binded before!' + done[str(i)] = 1 + else: + if not havaException: + done[str(i)] = "delete failed !" + success = True for k in range(len(dataIds)): if not done[str(k)] == 1: From 67db39702d491e2aa34baa6981085fc74af25741 Mon Sep 17 00:00:00 2001 From: JerryYin Date: Thu, 8 Nov 2018 20:06:11 +0800 Subject: [PATCH 05/42] done getFilesInfoOfPath() fun --- .idea/workspace.xml | 329 ++++++++++-------- .../config/__pycache__/config.cpython-36.pyc | Bin 1655 -> 1683 bytes project/main/config/config.py | 42 +-- .../__pycache__/fileManager.cpython-36.pyc | Bin 2565 -> 3994 bytes .../__pycache__/projectManager.cpython-36.pyc | Bin 5837 -> 5823 bytes project/main/manager/fileManager.py | 115 +++++- project/main/manager/uploader.py | 43 +++ .../__pycache__/httpServer.cpython-36.pyc | Bin 4704 -> 4772 bytes project/main/server/httpServer.py | 2 + 9 files changed, 356 insertions(+), 175 deletions(-) create mode 100644 project/main/manager/uploader.py diff --git a/.idea/workspace.xml b/.idea/workspace.xml index 21324d6..bde00f6 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -2,9 +2,11 @@ + - + + - + - - - + + + - - + + @@ -47,8 +49,8 @@ - - + + @@ -57,7 +59,7 @@ - + @@ -65,7 +67,7 @@ - + @@ -75,17 +77,15 @@ - - - - - - - - - - - + + + + + + + + + @@ -93,8 +93,8 @@ - - + + @@ -105,35 +105,48 @@ - - + + - - - + + + - - + + + + + + + + + + + - - + + - + + - + - - + + + + + @@ -141,8 +154,8 @@ - - + + @@ -150,22 +163,26 @@ - - + + - - + + + + + + - + - - + + - + @@ -182,13 +199,10 @@ - getNotebookName - ns_host request & params not support!!! users createPreProject jupyter.executingNb - executingNb ns_doname file_system_readme projectName @@ -212,6 +226,9 @@ resp_err_version_invlid dir_home custom.css + executingNb + createProject + unknown @@ -240,10 +257,11 @@ @@ -492,9 +510,9 @@ + - @@ -525,13 +543,13 @@ - + - + @@ -581,28 +599,7 @@ - - - - - file://$PROJECT_DIR$/project/main/server/httpServer.py - 187 - - - - - - - - - - - - - - @@ -631,20 +628,6 @@ - - - - - - - - - - - - - - @@ -743,13 +726,6 @@ - - - - - - - @@ -769,13 +745,6 @@ - - - - - - - @@ -847,31 +816,40 @@ - + - - + + - + - - + + - + - - + + - + - - + + + + + + + + + + + @@ -882,76 +860,129 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - + - - + + - - - - + - + - - + + - + - + - - + + + + + + - + - - + + - + - + - - + + + + + - + - - + + - + - - + + - + + + + + + + + + + + + + + + diff --git a/project/main/config/__pycache__/config.cpython-36.pyc b/project/main/config/__pycache__/config.cpython-36.pyc index e1eefb30725752267bc601487cf769066dd4ea0e..c91c8c3faa616f55126f6c15c43aa51aaac27ac9 100644 GIT binary patch delta 363 zcmYk&yGjE=6b9g#y(HOW$K12IdMOO18Fn|iDhT=nT3852Bp_$NMg=25(pVS}>||fV zPD?AF!cx#epTO2%E9Wr(_YbF=Z$DMnL#6Vs-pApS65?I_J2v_b?)VwOf&@#FY>~w_ z+3b+RF1f~`3}-0IS;}#aJoZdB5S$N$$s6xR27(G)AfJ6w#%G219M4e^y@*~iC?%9K zEW-+{z$&c58mz%OtiuLuzyJoY$xUv}1hu%$9oXhB_h5(Tc>#8LabFCU?%%eTqd3y5 zp-y7G5l;Oi)KRRXVWQVIr(P7V>ktz?ZpjBro~2**m7Jx=&Pd*-FV0c*?)M`sgpd~j oa(B>6W4EvT$?14>az4Hsv6-X??q?;z){&UfUrMK+?qybf0L%VL6aWAK delta 335 zcmYk%y-ve06a`>AandwDZQ8VTe}M_;5;c@Y?E;KU46t2<5Yo+rM5VGI7#IKvi3#~N z*jQM30Um%AB%T1Va@eqS&v&jjo9HLJ9!HVCfAxO&8VT_s{v8{A2Xp>{U_pWpZhkM+IT^{ff?D6t}7_K~iZV&QNnr)=n zI8P_D%W5`5&!=fN#f&D;yYk7BbMtLq$UAeUj^(|1Q%A|e-{&oakPD668}`kXvuYO3 SSG&Q!f>ZZZQ4m@|pdF{7jzu>A)A@jUL>eIztcM-#9is${Qb$R)vr8aTu$7AKwpS zjX%Q=z&OI`GHFhH^RP$h^>ji%u&_!J?e?oN>FSbf(bMGW{1rXX6C*Jt^+^9Tr?DaR zExK++N2S&^r7jI=a%~4>lx!YNuW2(yOEB<4I0$a~u@=zL^iS;(&C;LrIeIKLjR`uD zP8;Fi4fNImAk?LdiYZ_nigWml!0W<$3f|~9kV?YIx)vP<_h?r`2R*VoQ|@|19N}b{ zfZjlDLM6o=;x#3O5Vk0%FkcE_6@)s_0CaebZF>7|Y{hB`#=T9iu3^m594CpzwZx8X zG}&bwA{9IN4A9!54U*VlJ(qC3L7qsJqmHz}6{fhWxTAPM+5=FyYT_wHLGdE;Z5oww zL){WLFw$)*X1S5e)-9Y#OWLv^UFk^1%^fT8;7ZEUQ&-Vdw)EVV_>Nn{ok=mSsMAXE zpejqanA4;Tu>>J}6624Wp2g3g}GE#vvOolccguH}Abez?EG` zWLZ{ZkvlsqsY)H(sqVT{%^l!w19nm(jc$2>>4UQAf&mRni5GVCq$X?2aMMtk-jcv# z$jD#xg^bQOt;`79J-=mb^gHd?&x|Oz>t}8p^!!u8M~UK#eIBgb$@z&+$B!aW1v9^* zEl~$(GVvVp!4bqJ!cm0h0a{jgvv)cW8H4f7V8hQh-z`iA(d5l=J?!5Ko4PV1utGC? zD6BnQ_NXy~h=U<}BZy)o`i387rf7$&z8J*?1LlW3x)sE08C&(^HU^?t?=kBrW`kD6 zY6Z~>_>{R(+!nFgDjHx)KrbL-5}-LEJQy<8>4&i&#?cd3#XKr#kkL-Y(9GasJ6=;! z=kcpdsGRJBl`R%T%XgyK?`1|1uJl`Wo|#C@fmVzoj3JaBLm60B17}s@&cdb(D|p7+ zAdk8LgdV1b`qmBV(Se~EFg55XeEbG2(VAA*JW${>Z)$Y-FwN6Sx~H+-bsp?<>8L)` z=OeDF(2|BTris_#xDNJ2{NafAubPPg1Pg4?q)Wxf^{kIZ}Lp<%*z%(2;-+-bl^-XPq9mPoqR7E$9~x&0q zTJKutVBU1nsy#iPIbn2-_c3Ex8G>8t$Nr6ee|;%!+lQvl!$PqLkQwov_k3|2=NA#q zAe==wmwsg*n)?xG^ejOBC*(q9xpbd;YHV0xV|0ujv_vcY+y0Esrk}ENFV3Qg%K%MH zTtR*nxxJ?g2rLV631Kp=JFn2+(?w^$a~;>xOU-(^=^SrPpn3`ci&mADz*`q55%4NR j3*mbDgEMn@5-8jKO5jV-*@#Yx15cSx*i<4|NU(iG-N@+vWZcti2q>4iVRrwI&5CzHsg%;Fymo!O}fOjjE z%Hcr7v4_ex-~_*be}MQIxgof~krU#?I8*{5Jeud{H#6R+^{eTZCySTj@#s!4esGW^ ztMKC$$WRw6Q6N~s!G-jgx zi1?uvV$UCjX2+7ccw$7*Cj%luqGOMv#=S_ENFz%;D|_5C+5goeBl2H8s=miu_7j%W z8M{cog1TYfx60_sjj4C`+&Q7$He>$2zr7c>`>p!cIMMRM`XKbT27_*Wv*(W^{os)| z=&MimOnDL8#S+5U2%kOioBVNm*KcfwJL00QENE~wl+}EE##%=joK^MsRK`p*3cn)< zIVf17s`lcqp`udG?Q})Y)DT=Mu4sN$^_Z1U5bf&u+@6n}nS?A3Mj1z7#Y((u1r#oDh_E?F7|9M%NY>kvWOw6Pd!~)- zRZlEs{=i7h1<6Q0a|_D}A;A$MxFAkkK>UD+3z6V}sM(FNm{N6hRXvqnz3Cs9zgcE) zcpeKEf4lX({e*>@;}ZP9M!Pz>q#aYZsJ=JW;0x6>e}d1|?`EcA%a}8z*CHHRqy!)m zB$bYULC7t2*J{0EwTN_QeXN=#X^~m?e|4*_bNk3#C#f!?b;8LyS!$>+t@zqPZzql) zb=rfyodc0%o!(w9M%yn!yO-zjIO?P$k!+1d!@Psdtb0G1{A#@q=5^d4e^b5HxM*wV zp|)spr|~HTuKseKgHnIEuUwB1bO!z18f|fOjD{ZSA%_U235d;|OnKyU%l`sblY1PW$%^KL!E+PL}uz|x( zY0os9W>%eXbcu@9c-|0()TB9w8Fna6ZAfF_%0}JYJ-S9K!1A**Qi z7)ap*87~Td&=i66ONwh57(p_OW(jW-9v00Faua_bNAx3qlcaR>WO=8Aisr|ho#N9- zHa|kSS9o+a~<9l-A;axi)4Fx&l8b%#^&~r_p@025Ukx;qFDpqK*sPG+b5v+#%8u1 zFK6)w<0elYCL)RNb^8w!9xsev-b=UoDUa*f^}IOW>-Ca6U&sJwsI4%8NPQRHTKLd= z(ET`B+wUbiqCZN1fn-|g{#Q|p(aP(e)i3Za9t2FP!&%VL7+llZdO%%ZnhU<64x?9F ox(*Z~8i5M|oTI+xQxo&alZa{X=j7{`OmIB=g8dq6+l+1g3t0E^eEUZ~dJ z^Ri|;1P2&68PvPSwbvAmsZaSCc&%*q1)i&KEU+X=#y*kaRL|RoMpi0on z5|W+h{l~8N>_YFbIpXJ{${Z17j*J-Uqwbn~!f!=hqt#jqyF6ZbIH`idFr!0& zv)d`Wps=jYSU=&J8nX{wDq%<&*QH+QtVW2;r4Zh_N8)@6n^O^Dky-)?TqY?j9*>!T zSjS~Vw5bVL(}GkzwNHCm0n zRT6XL@#a4sX~x4u9@ZX@6oTK#WKw-~ruAZ##FoE;Q0r=@cy=q(ck!5rhlWo?Nphsg z5;#wMxz7hd^Km`HSeE7vwp|Uon<2ISX3i7;LXJfC8c%8P3rK7OX%9zEvlx}6olty* zytKCx*q?F-n=UZ)iG7>+^foqTy*W=rC3ZIJ`-v_dj*D4dF@ZrMtTmcyVMQE>om5+| z%oB-ca8X#IZ|AWDlYCd|eX9Oz!2te>z$EH^X<)iM5k3wh@4x z7|p>!$l>I08b;|j3{wlZ=1>X-RjtaAe(F#M9H6)yC_x1*I04 1: + numDirs = len(info) - 1 + numFiles = 0 + for i in info: + numFiles += len(i['files']) + + d = { + "name": c, + "size": size, + "timeCreate": timeCreate, + "timeModify": timeModify, + "timeAccess": timeAccess, + "numDir":numDirs, + "numFiles":numFiles, + "system": isSystem + } + dirs.append(d) + else: + print(str(c)+" is unknown") + unknown.append(c) + result = { + 'files': files, + 'dirs': dirs, + 'unknown':unknown + } + return result + + # content = contents[0] # all files & firs of the current dir + + # return content + + +def moveFile(fileName, dir, dirTo): + # todo + + pass + + +def renameFile(file, newFileName): + # todo + + pass + + +def getFileInfo(file, path): + # todo + + pass + + # # get the directory number int the 'dir' # linux : @@ -103,7 +209,7 @@ def createProject(path, projectId, projectName, type, filePath): # shell.execute('mv ' + config.getNotebookName() + ' ' + str(projectId) + '.ipynb') # shell.execute('mv ' + config.getH5Name() + ' ' + str(projectId) + '.ipynb') - jupyter.executingNb(path, config.getNotebookName(), type) + # jupyter.executingNb(path, config.getNotebookName(), type) return { 'projectId': projectId, @@ -139,7 +245,6 @@ def deleteFile(path): except Exception as e: return (0, 'Delete Failed, cause: ' + e) - # for test # if __name__ == '__main__': -# getAllFiles('/Users/jerryyin/.jupyter') +# getAllFiles('/Users/jerryyin/notebook') diff --git a/project/main/manager/uploader.py b/project/main/manager/uploader.py new file mode 100644 index 0000000..804fc72 --- /dev/null +++ b/project/main/manager/uploader.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import os +import socketserver + +baseHome = '' + +class FileTransportServer(socketserver.BaseRequestHandler): + + def upload(self): + conn = self.request + s = str(conn.recv(1024), encoding='utf-8') + fileName, fileSize = s.split('&&') + newFileName = os.path.join(baseHome, fileName) + if fileName in baseHome : + #file already exists + recvdSize = os.stat(baseHome).st_size #calculate temp file size + + conn.sendall(bytes(recvdSize, encoding='utf-8')) # process + + with open(newFileName, 'ab') as f: + data = conn.recv(1024) + f.write(data) + recvdSize += len(data) + + # if recvdSize < int(fileSize): + # # continue upload + + + else: + #new upload + recvdSize = 0 + # conn.sendall(bytes()) #reply + with open(newFileName) as f: + data = conn.recv(1024) + f.write(data) + recvdSize += len(data) + + + + def download(self): + pass diff --git a/project/main/server/__pycache__/httpServer.cpython-36.pyc b/project/main/server/__pycache__/httpServer.cpython-36.pyc index cb136a5e11b6794ba899eac07c719f6b6b4a0a4b..802809a14f748fad577bbedbabe5f969322dcda4 100644 GIT binary patch delta 276 zcmaE$vP6~Bn3tEUR_ICew2hp8Jd9T+NAPS`t6@yx&Sok)R>GUbSHqCPBgxRr7|f8u zo5GjEKZgm(VhCo?6qqc{YcE*ES&~{@qMMePlX{D#Ah9H4aulyIBmd++UMI;SHUjzHFUT({ tiO(o0DJb#+39$o(GV+T{iade5BJau4LOR?wAST#cpUFW&{>*Y5(g43pG-dz* diff --git a/project/main/server/httpServer.py b/project/main/server/httpServer.py index 4ae4107..1a919d8 100644 --- a/project/main/server/httpServer.py +++ b/project/main/server/httpServer.py @@ -217,6 +217,8 @@ def praseData(request_body): return bindDataWithProject(request_body) elif action == 'deleteDataset': return deleteDataset(request_body) + elif action == 'test-file': + return fileManager.getFilesInfoOfPath(request_body['path']) else: return {'status': 0, 'result': 'request & params not support!!!'} From 1a6e9a53f50aac03d2d3fcd4bb7678f43d502481 Mon Sep 17 00:00:00 2001 From: JerryYin Date: Fri, 9 Nov 2018 13:48:21 +0800 Subject: [PATCH 06/42] done file manager's fun --- .idea/workspace.xml | 211 ++++++++++-------- .../config/__pycache__/config.cpython-36.pyc | Bin 1683 -> 1683 bytes project/main/config/config.py | 2 +- .../__pycache__/fileManager.cpython-36.pyc | Bin 3994 -> 5106 bytes project/main/manager/fileManager.py | 179 ++++++++++----- .../__pycache__/httpServer.cpython-36.pyc | Bin 4772 -> 5909 bytes project/main/server/httpServer.py | 92 +++++++- 7 files changed, 334 insertions(+), 150 deletions(-) diff --git a/.idea/workspace.xml b/.idea/workspace.xml index bde00f6..35b377c 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -2,7 +2,6 @@ - @@ -20,28 +19,28 @@ - + - + - + - + - + @@ -50,7 +49,7 @@ - + @@ -59,7 +58,7 @@ - + @@ -67,7 +66,7 @@ - + @@ -77,14 +76,10 @@ - - + + - - - - @@ -94,7 +89,7 @@ - + @@ -105,13 +100,15 @@ - - + + - - - + + + + + @@ -129,8 +126,8 @@ - - + + @@ -142,8 +139,8 @@ - - + + @@ -154,8 +151,8 @@ - - + + @@ -166,11 +163,16 @@ - - + + - + + + + + + @@ -180,7 +182,7 @@ - + @@ -199,10 +201,6 @@ - request & params not support!!! - users - createPreProject - jupyter.executingNb ns_doname file_system_readme projectName @@ -229,6 +227,10 @@ executingNb createProject unknown + getFilesInfoOfPath + split + deleteFile + dele @@ -271,7 +273,7 @@ true DEFINITION_ORDER - + - + - + - + @@ -566,6 +568,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - @@ -853,13 +872,6 @@ - - - - - - - @@ -881,36 +893,37 @@ - + - - + + - - - - - + - + - - - - + - + - - + + + + + + + + + - + + @@ -918,28 +931,27 @@ - + - + - - + + - - + - - + + @@ -948,41 +960,58 @@ - - + + - + - - + + + + + + + + + + + + - - + + - - - + + + + + - - + + - + + + + + + diff --git a/project/main/config/__pycache__/config.cpython-36.pyc b/project/main/config/__pycache__/config.cpython-36.pyc index c91c8c3faa616f55126f6c15c43aa51aaac27ac9..5b886fa3d9e59532fd081fb079ce7b4d7de1362c 100644 GIT binary patch delta 22 dcmbQtJDHcwn3tDp;jfKsyo`*7n?)GwSph<21$+Pi delta 22 dcmbQtJDHcwn3tF92IodLUPeZP%_5BTtN=OQ1i=6R diff --git a/project/main/config/config.py b/project/main/config/config.py index 4d303c9..800d774 100644 --- a/project/main/config/config.py +++ b/project/main/config/config.py @@ -1,7 +1,7 @@ # global config file app_name = "极算云notebook管理分发系统" -version = "v1.0.0" +version = "v1.0.1" author = "JayYin" # server diff --git a/project/main/manager/__pycache__/fileManager.cpython-36.pyc b/project/main/manager/__pycache__/fileManager.cpython-36.pyc index eba30eca2bd584367643d9795bd804e7195af898..60653bcbe7609ed04d667b0317f3001d03e21cf3 100644 GIT binary patch delta 1965 zcmZux-HRMW5bx@lotd4T{kY8~m)xDXSuqJK$%BNvoFRtf58^Bq0ychadq_{|3Q7zz11GP%tRI2_gmrt9m9Ma?a32^{=X{tE+z9 zpDg@w#6Mmx`w#42-u~vNI0)3ddO-$tq9(vfr6{pd3}uu#Nu`+kxxfr8f#^ z#yha}rg9aFAuei{jIl55Z4hvPjlA4a+GQ!`mSC5~H?hiuk4i?bkiG-SN#$w><5wwu zfGNmp;)b1pJ%^^+(XK9OUl+9}3b&n6pm|)=!8QQv-!s9iESUUNIFDA2%4*7NS5b9c zAzuNbDxMW};r48dd^c@5$KZbYkux{0=_wO$R@Jiz!k=M;XdCy!1c9xv4`Q3CJw1Pi z%TvHcbzsAJH68C&`*eMq|JU~_6Zntnx9Y37j>ZbpTwB@}rQ6nMkKVJ4_GQ)#E=pkO z@6J7#Nk4YaIfZ0%t*g@S-P7}v;<=mcXj64J`qILtO{_>t>7v%0PG`N?nn09(=p6%@ ze&s#qfeo|tr$2ZhK>Ck2SA{wR5ONM2JTA|sjpCK@r__Fqf?0A2W!vbeP~aL4!;csk zquMGiKv^1Vf%U*M<`_o|X^~+XPtxOZQCPyp4)JBC!5|k9aWX{NaUpgv&KQzdSXoHn z&am8Uwiq$ZVpn*`60SVWu~o(WY=pMBNT|G%qrO40oXAHzTZ0jiLDV!sXh9c*H-m}H zuo2{&W5V*m*ldofC_UPI>kK;h8lBAvA7z@+asJ6-w}J^Gk7maCiIE)6EoYVW`D#Xl>cwFpk?qBBSP+R?43QR$ z4fig~cGzz9!~TW}-;2WjU~MgIH2K6yR7(vKx!-s#C2F_VFfgk+8-u|@2E-z{kAz2}NWv#^g2W39_m{^}oj6%o`6A&bQ8e9&g$0;cc>D<@@)Q}4kT5~c zkXCsts-(S);y#Xqj0~pn_mJ%#&t!ml&GNv1VHk1xc#ILV0~h$3EIOW}MQdM=W}UuIi%1u+)|&9a9>l$IHi!s_|*8d3rukYP=L z@W!~z+?)>H6C?rLf~QDeUIWG+&MkFi)K(jEYj;~sBOlH`@#t*Gb<^o&u(`P*6}HG( z?}@Gq!;xyhPK>BswIz>Fd>(N^f{Y?}Rt)mAu3M0v@rd?EnA( delta 853 zcmaiyziSjh6vyYyZT9Eg?(Ll*CPflRloL~lg$Hu+H0FmfQB;D7!AN$Ew~3cHSux6- z0q+k;a5_n&NMa)*+9+CE+g@R1BWPn|C%!jlP_&R?zVn&)-hSWk*l!0OcDsjjIk(pT zX=bGBh5OM6ii|NA_X*sqUqF2~u7r0}GwqPj#C4Kw!t8am&{17aSptb8ttK>V&8}ug zv!~hBoYm}U&Jl0Is-IfYY(nBkMia7VDh@?PJGN-jO`6h@o^+)n?WkkRY-A-*0-0?B zu$u{b9fc&*HWP0mbNACmF3sz9ei}-jiX&(Q_Q%rK1kw)ZgWf}!pP^Ya|P(AWfU6Cs%i$b2*j!Z9Yv 1: - numDirs = len(info) - 1 - numFiles = 0 - for i in info: - numFiles += len(i['files']) - - d = { - "name": c, - "size": size, - "timeCreate": timeCreate, - "timeModify": timeModify, - "timeAccess": timeAccess, - "numDir":numDirs, - "numFiles":numFiles, - "system": isSystem - } - dirs.append(d) - else: - print(str(c)+" is unknown") + try: + timeAccess = str(os.path.getatime(filePath)).split(".")[0] + timeModify = str(os.path.getmtime(filePath)).split(".")[0] + timeCreate = str(os.path.getctime(filePath)).split(".")[0] + size = os.path.getsize(filePath) + if os.path.isfile(filePath) or (not str(c).startswith(".") and "." in str(c)): + # is a file + # files.append(c) + # get file's properties + file = { + "name": c, + "locate": filePath, + "size": size, + "timeCreate": timeCreate, + "timeModify": timeModify, + "timeAccess": timeAccess + } + files.append(file) + + elif os.path.isdir(filePath): + # else: + # is a dir + isSystem = "system" in str(c) + + info = getAllFiles(filePath) + print(info) + numDirs = 0 + if len(info) > 1: + numDirs = len(info) - 1 + numFiles = 0 + for i in info: + numFiles += len(i['files']) + + d = { + "name": c, + "size": size, + "timeCreate": timeCreate, + "timeModify": timeModify, + "timeAccess": timeAccess, + "numDir": numDirs, + "numFiles": numFiles, + "system": isSystem + } + dirs.append(d) + else: + print(str(c) + " is unknown") + unknown.append(c) + except Exception as e: + sysout.err(TAG, e) unknown.append(c) result = { 'files': files, 'dirs': dirs, - 'unknown':unknown + 'unknown': unknown } return result @@ -163,17 +166,85 @@ def moveFile(fileName, dir, dirTo): pass -def renameFile(file, newFileName): - # todo - - pass +# +# rename file or dir +# +def rename(src, dst): + sysout.log(TAG, "src: " + str(src)) + sysout.log(TAG, "dst: " + str(dst)) + if src == None: + return { + 'status': 0, + 'result': 'Src can not be null !' + } + if dst == None: + return { + 'status': 0, + 'result': 'Dst can not be null !' + } + if not os.path.exists(src): + return { + 'status': 0, + 'result': str(src) + ' not found !' + } + src = str(src) + s = src.split("/") + i = len(s) - 1 + if src.endswith("/"): + i = len(s) - 2 + name = s[i] + path = src.split("/" + str(name))[0] + dst = path + "/" + dst -def getFileInfo(file, path): - # todo + if os.path.exists(dst): + return { + 'status': 0, + 'result': str(dst) + ' is already exists !' + } + try: + os.rename(src, dst) + if not os.path.exists(src) and os.path.exists(dst): + return { + 'status': 1, + 'result': 'rename success !' + } + else: + return { + 'status': 0, + 'result': 'rename falied !' + } + except Exception as e: + sysout.err(TAG, e) + return { + 'status': 0, + 'result': 'rename failed, '+ str(e) + } - pass +def makeDir(dir): + if dir == None: + return (0, 'Directory can not be null !') + if "/" not in str(dir): + return (0, 'Directory form not support, should be absolute path !') + else: + try: + shell.execute("mkdir " + dir) + return (1, 'Create successed !') + except Exception as e: + return (0, 'Create directory failed, ' + str(e)) + +def createFile(file): + if dir == None: + return (0, 'File can not be null !') + if "/" not in str(dir): + return (0, 'File form not support, should be absolute path !') + else: + try: + shell.execute("touch " + file) + return (1, 'Create successed !') + except Exception as e: + return (0, 'Create file failed, ' + str(e)) # # get the directory number int the 'dir' diff --git a/project/main/server/__pycache__/httpServer.cpython-36.pyc b/project/main/server/__pycache__/httpServer.cpython-36.pyc index 802809a14f748fad577bbedbabe5f969322dcda4..d5045386dc6117a2ce8d259cf04a5809b3315be9 100644 GIT binary patch delta 1228 zcmbVL&rcIU6yBNLmbG-3Z384qkwT?d6_6i^0RbUU6fr0uBq9kZ%wlVyOJ>IdWE*1e z=0!8{VobbvHO3e(o;-Lp;cDZ-Xz&lXZxq-zWD03%RBWr@bjwRSB=wqyzC84ipbXrNZy#AW4 zG&NWg0UO4(SvjGnH$AdNimInFB~SSrqfBMmGg!d_5Zm()#71)~ zcHH4{mgDx4aQJ$>;E4FLbFaV}Q+{CF=yD?$fCbs1{Vy9n?B%&i4bh(;(06N|)1i0G6OU8BU$Swgbd4&8ee z0!%3qFuE@LDu=EJ^{(a-k8Yt->O@I(`BF(`t~i2uklTgKs{NB}A8g3`0pxb5re^@0 zPhc+I6v-hyeV-u5*)pvF49ex0Etg}S7S*EWsh-9tmI5gGf*cg~2xcK>uh_!I@<7Hf zQpVu8Kk9i2CccBYRy*K9K~0d8NA(%{+ai@Lgf; zMOYQHSj7Jm_!FYC!$@?_RW9ZVY$PKxt}S>sQtmr0{&*UfE~1%LOwO#K+CkZyfVE36A0pmu0opP117ATBN6md}+0j zN%`5@2u|YcRXHE&uN}rc@VhA#zX+cJFLhGBiqw;F`7zScG6I28xZp0VIb1Acg;?X$ z7^;9!)^P=&f)T$d>!Rlx4&Tcs%F{UQA2=hYqDgXIu1AxGrPt_&9@Z^Asn`7g4dUcv delta 396 zcmbQLw?tLhn3tEUR_ICeG#Lhl#|%h-8OU}3;^JKsm3Of+1v6;MZ)Rf5;GUc=z{_-n zVRD&3HzW6CCP7{KVjWjChA5&%ZkiFge8bz0}@d}>8T}dK*hzL zd1?9nX#pS|kwR*0hk%mBdXrm)^jY|riflI@5t`4)W(U${KRHvxgwb~LJdq@(M+}ph zMD-XwCmV`Zv3Y`IohEM-wKe$*(hV}7fr*DvfU(FI$kLRG5-l%I&n!w!(<{!*EyziY zFHS8gOD!s50?HOSPgW9BSGNH&Z}H?6#~0)mmBeS1loS+sgM`?DLK*qRB}HB!wLX&z u#B>B~K}@ivAmfWfCO;Gr=I5}<%}*)KNws4IiWO^2ek7*G%EzI=Aq@b*%3;<3 diff --git a/project/main/server/httpServer.py b/project/main/server/httpServer.py index 1a919d8..39ddd19 100644 --- a/project/main/server/httpServer.py +++ b/project/main/server/httpServer.py @@ -190,7 +190,83 @@ def deleteDataset(request_body): else: return projectManager.deleteDataset(userId, dataId) +# +# get all files & dirs info of the current path +# +def getFilesInfoOfPath(request_body): + path = None + try: + path = request_body['path'] + except Exception as e: + return str(resp_err_params) + str(e) + res = fileManager.getFilesInfoOfPath(path) + if res == None: + return { + 'status':0, + 'result':"File or directory not found!" + } + else: + return { + 'status':1, + 'result':res + } + +# +# rename file or dir +# +def rename(request_body): + src = None + dst = None + try: + src = request_body['src'] + dst = request_body['dst'] + except Exception as e: + return str(resp_err_params) + str(e) + return fileManager.rename(src, dst) + +# +#delete file or dir +# +def deleteFile(request_body): + path = None + try: + path = request_body['path'] + except Exception as e: + return str(resp_err_params) + str(e) + (code, res) = fileManager.deleteFile(path) + return { + 'status': code, + 'result': res + } + +def makeDir(request_body): + dir = None + try: + dir = request_body['dir'] + except Exception as e: + return str(resp_err_params) + str(e) + (code, res) = fileManager.makeDir(dir) + return { + 'status': code, + 'result': res + } + +def createFile(request_body): + file = None + try: + file = request_body['file'] + except Exception as e: + return str(resp_err_params) + str(e) + (code, res) = fileManager.createFile(file) + return { + 'status': code, + 'result': res + } +# +# +# +# def praseData(request_body): if type(request_body) != type({}): return resp_err_params @@ -209,16 +285,24 @@ def praseData(request_body): return runWithVm(request_body) elif action == 'deleteProject': return deleteProject(request_body) - elif action == 'getMyFiles': - return getMyFiles(request_body) elif action == 'bindDataset': return bindDataWithProject(request_body) elif action == 'unbindDataset': return bindDataWithProject(request_body) elif action == 'deleteDataset': return deleteDataset(request_body) - elif action == 'test-file': - return fileManager.getFilesInfoOfPath(request_body['path']) + elif action == 'getMyFiles': + return getMyFiles(request_body) + elif action == 'getFilesInfoOfPath': + return getFilesInfoOfPath(request_body) + elif action == 'rename': + return rename(request_body) + elif action == 'deleteFile': + return deleteFile(request_body) + elif action == 'mkdir': + return makeDir(request_body) + elif action == 'mkfile': + return createFile(request_body) else: return {'status': 0, 'result': 'request & params not support!!!'} From ffd7ee9e69a351cae466331ee2b7f6a2a47b016d Mon Sep 17 00:00:00 2001 From: JerryYin Date: Mon, 12 Nov 2018 15:39:32 +0800 Subject: [PATCH 07/42] done move file to dir --- .idea/workspace.xml | 258 ++++++------------ .../config/__pycache__/config.cpython-36.pyc | Bin 1683 -> 1683 bytes project/main/config/config.py | 11 +- .../__pycache__/fileManager.cpython-36.pyc | Bin 5106 -> 6046 bytes .../__pycache__/projectManager.cpython-36.pyc | Bin 5823 -> 5823 bytes project/main/manager/fileManager.py | 39 ++- project/main/manager/processManager.py | 18 ++ project/main/manager/projectManager.py | 1 + .../__pycache__/httpServer.cpython-36.pyc | Bin 5909 -> 6519 bytes project/main/server/httpServer.py | 16 ++ 10 files changed, 152 insertions(+), 191 deletions(-) create mode 100644 project/main/manager/processManager.py diff --git a/.idea/workspace.xml b/.idea/workspace.xml index 35b377c..7a494b7 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -2,9 +2,11 @@ + + - + + - + - + @@ -49,7 +52,7 @@ - + @@ -57,16 +60,16 @@ - - + + - - + + @@ -76,8 +79,8 @@ - - + + @@ -85,52 +88,29 @@ - - - - - - - - - - - - - + - - + + - - - - - + + + + - + - - - - - - - - - - - + + - - + @@ -139,8 +119,8 @@ - - + + @@ -149,42 +129,14 @@ - - - - - - - - - - - - - - + + - - - - - - - - - - - - - - - - - - + + @@ -201,7 +153,6 @@ - ns_doname file_system_readme projectName resp_err_params @@ -211,7 +162,6 @@ system .system shell - bindDataWithProject startVm sys path_dataset @@ -231,6 +181,8 @@ split deleteFile dele + bindDataWithProject + makeDir @@ -259,11 +211,13 @@ @@ -273,10 +227,11 @@ true DEFINITION_ORDER - + @@ -315,20 +270,6 @@ - - - - - - - - - - - - - - @@ -336,27 +277,6 @@ - - - - - - - - - - - - - - - - - - - - - @@ -373,6 +293,9 @@ + + + @@ -542,16 +465,15 @@ - - + - + - + @@ -626,13 +548,6 @@ - - - - - - - @@ -897,9 +812,6 @@ - - - @@ -921,10 +833,6 @@ - - - - @@ -932,9 +840,6 @@ - - - @@ -942,9 +847,13 @@ - - - + + + + + + + @@ -952,66 +861,61 @@ - - - - + - - + + - + - + - - + + + + + - + - - + + - + - + - - + + - - - - - - + + + - + - - + + - - - - - - - + + + + + diff --git a/project/main/config/__pycache__/config.cpython-36.pyc b/project/main/config/__pycache__/config.cpython-36.pyc index 5b886fa3d9e59532fd081fb079ce7b4d7de1362c..ad692cf6e924c6e1408e969f68d78a96e0ffed20 100644 GIT binary patch delta 71 zcmbQtJDHc$n3tEU>+6f?){UI6nHlpZe`T&sj9r^`SbZ3!xmfr) R7=e%piDc#w<6!1s1^^Dm5BUH9 diff --git a/project/main/config/config.py b/project/main/config/config.py index 800d774..5410fbc 100644 --- a/project/main/config/config.py +++ b/project/main/config/config.py @@ -77,20 +77,21 @@ #sshfs for storage on StorageServer #cmd_sshfs_mount_storage = 'sshfs -C -o reconnect user@hostname:remote_dir local_dir' # +sshfs_pwd = 'fd324;1' + # release # sshfs root@60.12.136.60:/sshfs/ /notebook/storage -sshfs_pwd = 'fd324;1' +# pwd: fd324;1 + # -# dev & localhost [private network] -# sshfs root@10.10.45.246:/mnt/ ~/workspace/notebook/storage -# sshfs_pwd = 'abcd#12345' -# dev2 [common network] +# dev [common network] # sshfs root@60.12.136.61:/mnt # sshfs_pwd = 'abcd#12345' # path of the user's dataset # path_dataset = '/file/datasets' +# sshfs -C -o reconnect root@60.12.136.61:/mnt /notebook/storage # the common base nb of the sys diff --git a/project/main/manager/__pycache__/fileManager.cpython-36.pyc b/project/main/manager/__pycache__/fileManager.cpython-36.pyc index 60653bcbe7609ed04d667b0317f3001d03e21cf3..5f468dc08d6cf032c325d34564495db0eb8454fb 100644 GIT binary patch delta 1684 zcmaJ=OK%%h6uxIXvB!?bah!+SCQ8Omli(&m1w?{Y5i|{{)f8xPh(bai?%c5w$B*dF zxJszuDVMQi&jP5A(S70REc%f?Wj8t0+t9iUI6D@KN3O9Nb~8t=iJ9R-~I0I zlV86&_PS|CMB|sW^$a^SibAf9BrE6EC4Cl8ok!F&+&$Pe>7OU+1ePv@b15O=+`Ua}9s zD(M&Jc_@iT1LHk8rZX?J84*7LyR8AM4|_&2>_v99eH>vKQQT9bGN?q3bM9z+E3Bfe z-~cs`zb^oLZVWDU8C(H-xw8o5lVxAbE zw%qkvPxH*#59sWc#+RF1>3W5@-V~BF6c-3c zKb+l^N>*Y+@Q3^&HpD-~xJ@RakEVBC(+wPW?bj4W`vpAk(72l+G0Y=5wv+I*Vt{w; zvX-5@m7R68^D5dI4PK9covGNW7D+Nuigf3@(cHG93~ z%EWo$lsRm!E#6X-EAc)e-qn;zS@Ay#D#tzNQvHanil9!g zcjhK=tESzeplkB0SQwV&H?jM@Ny-LQHjd^KYX=|qj6z7fiy?7R-i}YglzbRpfot-w zcp5IqRPXHgbxPhq@O5uz%MowW*e3Xd;3~m2x!;?Dw0zh*1ts~k_sc=*sS%rZN)D=% zQ_nReF3L|6Gg;Mc8S^N^MOs}U9{TrtJnVmzCMg>Qn(y^E1BrG;=#lV@!)$)q8`}E{`Nm_-pu^p%sxN+GoKmk=m>u` zzAWvgHSLr3-=m`*!PkAdmAMKhwpM5TFsIq6H+xe$DA{dHLrQYv5oC5g8y|d$Jm!i5 z!6uv{6y;uU7cBXNozI@ZfP$t1N3lk8m0zp1mqqb!syC@1f3P&1mR+IO&L|E!WP%?0 z0=^V<4Mbf-FHjLCYFih5_z@ArqQBl0)paj^fKH4m)>_}-W!!Zed#(?m>7umL8~F}_ zEOx|z44az^LnO};1`#SC_*$*S#YKu-LfAp2yTVa=-EAvy#bp}s3J;OGLLf)~b{1pU zeI=DS31jkYv`>z9KK6FpKQ7;Q_C&1yhk@1#dbZGl8NIMHiTSoqP_bg_Ko5|fCT)l$ zY4Wdve3Lu(+XwD&233EU0~Ku5R(Po?KFSx-UYL>Nv6o%HFxd?tG>C!;qPp`tmID7i zSRyOa@jML3iTDD{%k6j?D)L=?ID3oMUqmRQ{oo!KBhH?X>^`>r606}`{|T!i3$C$MydE6#RoQ%i?}MAAmek->E2_mD z29F4Zmm9%ByxoRqs~ER?gKpG}BiN4sRQYfjXR`VHU4kdpZrmL7A!+U;$pGq-8Y60_ zA1AOTXV-C%El-dBEbE#~r}okaeEjTZY-6$47kl04mQGw;!Y|q4Im@sN&nOt@{{eq+ BflB}Y delta 264 zcmexvG*wU8n3tE!hxKXnEF}hp#|%h-8OU}3;^JEqm2a^!1v6->Zx&)q<)2(3#LLJq zxlQP+Knj0~K#JfTpj-+AP+n;Adg0K?f+EI@f|FfE!X_^jQDb9b1e#zq`J#voGgFc4 zW(m>xjBIW|Q;Xau?-es)be;T3EQwKLa;UgIW8mag@hY}JkfhgSRtZ}%XOLbtAfJJW zhf#pBC=|%jl$#tOA>#PfD!`$}Aq@Z?e>~*? diff --git a/project/main/server/httpServer.py b/project/main/server/httpServer.py index 39ddd19..5eab12e 100644 --- a/project/main/server/httpServer.py +++ b/project/main/server/httpServer.py @@ -263,6 +263,20 @@ def createFile(request_body): 'result': res } +def moveFile(request_body): + file = None + dir = None + try: + file = request_body['file'] + dir = request_body['dir'] + except Exception as e: + return str(resp_err_params) + str(e) + (code, res) = fileManager.moveFile(file, dir) + return { + 'status': code, + 'result': res + } + # # # @@ -303,6 +317,8 @@ def praseData(request_body): return makeDir(request_body) elif action == 'mkfile': return createFile(request_body) + elif action == 'moveFile': + return moveFile(request_body) else: return {'status': 0, 'result': 'request & params not support!!!'} From 57eaa049cf6b5e0a8f5ccc290b540c9ac23e9485 Mon Sep 17 00:00:00 2001 From: JerryYin Date: Tue, 13 Nov 2018 18:18:04 +0800 Subject: [PATCH 08/42] done delete multi files --- .idea/workspace.xml | 220 +++++++++--------- .../config/__pycache__/config.cpython-36.pyc | Bin 1683 -> 1683 bytes project/main/config/config.py | 14 ++ .../__pycache__/fileManager.cpython-36.pyc | Bin 6046 -> 6248 bytes project/main/manager/fileManager.py | 11 + .../__pycache__/httpServer.cpython-36.pyc | Bin 6519 -> 6837 bytes project/main/server/httpServer.py | 16 ++ project/test/__pycache__/flask.cpython-36.pyc | Bin 0 -> 384 bytes project/test/flask.py | 21 ++ project/test/flask.pyc | Bin 0 -> 511 bytes 10 files changed, 177 insertions(+), 105 deletions(-) create mode 100644 project/test/__pycache__/flask.cpython-36.pyc create mode 100644 project/test/flask.py create mode 100644 project/test/flask.pyc diff --git a/.idea/workspace.xml b/.idea/workspace.xml index 7a494b7..f980e5c 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -2,11 +2,10 @@ - + - @@ -249,16 +267,6 @@ - - - - - - - - - - @@ -275,7 +283,7 @@ - + @@ -359,7 +367,7 @@ @@ -466,14 +474,15 @@ + - + - + @@ -548,27 +557,6 @@ - - - - - - - - - - - - - - - - - - - - - @@ -660,13 +648,6 @@ - - - - - - - @@ -864,16 +845,30 @@ - + - - + + - + + + + + + + + + + + + + + + @@ -886,36 +881,51 @@ - - + + - + - - - - - - - + + - - + + - - - - + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/project/main/config/__pycache__/config.cpython-36.pyc b/project/main/config/__pycache__/config.cpython-36.pyc index ad692cf6e924c6e1408e969f68d78a96e0ffed20..fe17b25940bbf38047174a85b466cf3cd2d0936f 100644 GIT binary patch delta 50 zcmV-20L}lC4U-KDGYt(2G^6TUJ`b@9>H`6Nll2350h5z%1YZG`lg+6f?R^E+Vub3J0Cx2zGU~HY7#}d!jG5H)zEMxa(9acL=MzzV= GY#IQx-4I9s diff --git a/project/main/config/config.py b/project/main/config/config.py index 5410fbc..98d41c9 100644 --- a/project/main/config/config.py +++ b/project/main/config/config.py @@ -57,6 +57,20 @@ # file_system_readme = dir_home +'/base/readme' +# ----------- DEV2 for 18.18.18.174 ------------ +# ns_host = '18.18.18.174' +# ns_port_http = 8100 +# ns_port = 8888 +# ns_doname = 'https://18.18.18.174:8888' +# ## ns_doname = 'http://' + ns_host + ":" + str(ns_port) +# +# # system home dir +# dir_home = '/notebook/storage' +# # dir_home_user = '/users' +# dir_home_user = '' +# # user's project dir: homeDir/users/userId/projectId/version/xxx-nb +# file_system_readme = dir_home +'/base/readme' + # ----------- localohst ------------ ns_host = '127.0.0.1' #_localhost diff --git a/project/main/manager/__pycache__/fileManager.cpython-36.pyc b/project/main/manager/__pycache__/fileManager.cpython-36.pyc index 5f468dc08d6cf032c325d34564495db0eb8454fb..448fd53086b57cf82b7ebcbf7cf509bfcd57cbaf 100644 GIT binary patch delta 279 zcmbQI|H44on3tC;amlOb2kHzAj~S2vGmz~7#Ko&7Dtj<0P0WepRY*~6VTckLaaWS5vqd~NQ1C5NQV)SNN1>Jtk9@oaAAm5sAVc)s9|bmjA5!}u4SoV%wnuz zu3<@GoC6eNWGLhcXQ%-g$>gWWbc-c9KPB}Rb8d0ECR0&9(4Zo5Afd^6i?txJB%`=U z0?50?SWr{|l5#8rk$iB1A0)}0lA4oRlIoV3lUf`CbYC$@o`I2tk&BTBNHQ_hG{q)A7t3cXnH(gp#;7v6Qe2gZg9iW#9ylog delta 78 zcmaE1Fi&6En3tEU=+}#ANp%K>#|%h-8OU}3;^IXUl|2|0Cgw!4G6pkfDsH?aA;zl7 W7@{dUd6jtnWH|{XMy1J)5~={B7!dRT diff --git a/project/main/manager/fileManager.py b/project/main/manager/fileManager.py index bc558e5..cd51b2e 100644 --- a/project/main/manager/fileManager.py +++ b/project/main/manager/fileManager.py @@ -337,6 +337,17 @@ def deleteFile(path): except Exception as e: return (0, 'Delete Failed, cause: ' + e) +def deleteFiles(paths): + result = [] + for p in paths: + code, msg = deleteFile(p) + result.append({ + 'code' : code, + 'msg': msg + }) + return result + + # for test # if __name__ == '__main__': # getAllFiles('/Users/jerryyin/notebook') diff --git a/project/main/server/__pycache__/httpServer.cpython-36.pyc b/project/main/server/__pycache__/httpServer.cpython-36.pyc index 694ea5c995bb2488ac7e1eb30c5a6f195febd716..b0da3d0dc763395ebf9204693bf2bb43798d36a9 100644 GIT binary patch delta 581 zcmZ8dzi-n}5PoMnuH`z3?Kp`IgdeS16KFvK8i@fZg~SqRRen^I5|AR$OG!ye!?OXY z1eQ`};8rAl42&>DjEIGQfHf;y$1begg9YKI_v!At`_A{hU#GszO`cAt&2R5_Z=MYq z`^oZAjF+Y#X{td^Mq- zM7x)g>{FF11C^_9q)uBJrTVv%PAsxC7vGtsDdAHjbb`@1Z}-|sJ6i9x4-`QOZCA1uU@P09_4CMFBC7PqLq|EoVosQAwPCot;M8bx0RDp&k+E175L;x-B~+?70C{hA#*w2Ed)J@)E?+BsU=2 z%Z$4`$31D2GqRe;&$*?7n5QA5)JB)Di>AKOT_bNaDC1SY!@74%m&+0H!7J;>dVf}x1U(9gdGeL=P?;@83%Xy9Jq4y@p{;z`1t;+DQ5ieeJ W6N6wHDp(CpjAsmsj%&;q`+oz{b$pHh delta 428 zcmYjMT`L4(6rM9{$I$LJA3JMitrE6vD2nw_sI|LT3Ku@}k*}d$tkpY1l3RBoom=t? zTy1jWZz#EwE3TEKVLybIp)Hiv9!TMU+JGgc3Rg4(FO)<4RC_U{Cd6+6moyVHc%Yp~Y0_XR z1ikwY(pZ0%*lHm4FaZH7=!qah02GLUB8tFx$2xWVnn=-4o*{Re%2&DtilaK3qhHcx zjzHIi@i9EEKKYB=PJzu!#qG_)&IZe3pV0+6y2c`8@xidA7~h)4ib&QcvRrv0at+c5 z$+A&=uSr4}X8!8qPniNr7w zPk~)`d9TF0<-F(Zu^CQP5|uBx9-F3&&El6eXz&lR91p)6xE2fJv!P=#8iFJqM_a1` LazNJPX1VqU!**TX diff --git a/project/main/server/httpServer.py b/project/main/server/httpServer.py index 5eab12e..7890c5e 100644 --- a/project/main/server/httpServer.py +++ b/project/main/server/httpServer.py @@ -239,6 +239,20 @@ def deleteFile(request_body): 'result': res } +def deleteFiles(request_body): + paths = None + try: + paths = request_body['paths'] + except Exception as e: + return str(resp_err_params) + str(e) + if not type(paths) == type([]): + return resp_err_params + result = fileManager.deleteFiles(paths) + return { + 'status': 1, + 'result': result + } + def makeDir(request_body): dir = None try: @@ -313,6 +327,8 @@ def praseData(request_body): return rename(request_body) elif action == 'deleteFile': return deleteFile(request_body) + elif action == 'deleteFiles': + return deleteFiles(request_body) elif action == 'mkdir': return makeDir(request_body) elif action == 'mkfile': diff --git a/project/test/__pycache__/flask.cpython-36.pyc b/project/test/__pycache__/flask.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..53e61f1dcb30083cea66113a6584325fc173320c GIT binary patch literal 384 zcmYL^u}T9$5Qb;>?vg{3#wXA+javv7A|eEWRhl4ySS)9DB+2dF9v+39RRfay$N(Ea8-3`CoZZ07T=Q1dYOE5TyO?PG=!9FXJJ-W^)& zPN$BnIr{(#-T~W@y7x$4t&pf`mZf@el=~{1qpF(WBBMf$U5$^<@wGdGi!@C;a#6HdLor4aVmZRvI<;a literal 0 HcmV?d00001 diff --git a/project/test/flask.py b/project/test/flask.py new file mode 100644 index 0000000..e5158ff --- /dev/null +++ b/project/test/flask.py @@ -0,0 +1,21 @@ +# from flask import Flask +# app = Flask(__name__) +# +# @app.route('/') +# def hello_world(): +# return 'Hello World!' +# +# if __name__ == '__main__': +# app.run() + + +import psycopg2 + +conn = psycopg2.connect(database="postgres", user="scheduler", password="jerryyin", host="127.0.0.1", port="5432") + +print(conn) +print("Opened database successfully") + +# psql -U scheduler -d postgres -h 127.0.0.1 -W + + diff --git a/project/test/flask.pyc b/project/test/flask.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c6f4563508a73fa25ee93feddf3b15bdb9b5a9b4 GIT binary patch literal 511 zcmcIgO-sZu6nyE&Dq6v#N4@oA{(y+Euz0cHmPOD*sLhjYn>Hm$m7do>>eZ`1KwrY* z!Mh=uc82L>UgNKmqYwOis$st(dfw2n_XLtQ0|n4Ccx1qk0Y{gw@HcHbJ61LFab9jK)AK?*eAc=v(Y&JJqtPwa zjytrDzxYa*s^ptM*hN?#)Tue$e lj=^K} Date: Fri, 16 Nov 2018 10:45:55 +0800 Subject: [PATCH 09/42] =?UTF-8?q?fix=20some=20bugs=20of=20fileManager;=20g?= =?UTF-8?q?enerate=20[/=E6=95=B0=E6=8D=AE=E9=9B=86]=20directory=20for=20us?= =?UTF-8?q?er;?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .idea/workspace.xml | 157 +++++++++++------------- project/main/config/config.py | 46 +++---- project/main/manager/fileManager.py | 162 ++++++++++++++++--------- project/main/manager/projectManager.py | 10 +- 4 files changed, 210 insertions(+), 165 deletions(-) diff --git a/.idea/workspace.xml b/.idea/workspace.xml index f980e5c..d588bd7 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -2,11 +2,10 @@ - - + - projectName - resp_err_params - runWithVm - config - getAllFiles - system .system shell startVm sys path_dataset - ln -s config.dir_home + config.dir_home_user + '/' + str(userId) pathDset path1 dir_home_user getDirNumber resp_err_version_invlid - dir_home custom.css executingNb createProject unknown - getFilesInfoOfPath split - dele bindDataWithProject makeDir ns_doname deleteFile + ln - + ln -s + system/datasets + getUserHome + dir_home + initProject + getFilesInfoOfPath + createPreProject + dir + dele @@ -232,9 +222,9 @@ - - - - - - - @@ -849,16 +832,6 @@ - - - - - - - - - - @@ -871,35 +844,36 @@ - - - - - + + - + - - - - - + + - + + + + + + + + - - + + @@ -910,22 +884,35 @@ + + + + + + + + + + - - + + - - + + - - - + + + + + + diff --git a/project/main/config/config.py b/project/main/config/config.py index 98d41c9..ccbf2ed 100644 --- a/project/main/config/config.py +++ b/project/main/config/config.py @@ -43,18 +43,18 @@ # ----------- DEV ------------ -# ns_host = '172.16.3.254' -# ns_port_http = 8100 -# ns_port = 8888 -# ns_doname = 'https://dev.dongxicc.cn/notebook' -# ## ns_doname = 'http://' + ns_host + ":" + str(ns_port) -# -# # system home dir -# dir_home = '/notebook/storage' -# # dir_home_user = '/users' -# dir_home_user = '' -# # user's project dir: homeDir/users/userId/projectId/version/xxx-nb -# file_system_readme = dir_home +'/base/readme' +ns_host = '172.16.3.254' +ns_port_http = 8100 +ns_port = 8888 +ns_doname = 'https://dev.dongxicc.cn/notebook' +## ns_doname = 'http://' + ns_host + ":" + str(ns_port) + +# system home dir +dir_home = '/notebook/storage' +# dir_home_user = '/users' +dir_home_user = '' +# user's project dir: homeDir/users/userId/projectId/version/xxx-nb +file_system_readme = dir_home +'/base/readme' # ----------- DEV2 for 18.18.18.174 ------------ @@ -73,17 +73,17 @@ # ----------- localohst ------------ -ns_host = '127.0.0.1' #_localhost -ns_port_http = 8100 -ns_port = 8888 -ns_doname = 'http://' + ns_host + ":" + str(ns_port) - -## system home dir -dir_home = '/Users/jerryyin/workspace/notebook/storage-dev' -# dir_home_user = '/users' -dir_home_user = '' -## user's project dir: homeDir/users/userId/projectId/version/xxx-nb -file_system_readme = dir_home +'/base/readme' +# ns_host = '127.0.0.1' #_localhost +# ns_port_http = 8100 +# ns_port = 8888 +# ns_doname = 'http://' + ns_host + ":" + str(ns_port) +# +# ## system home dir +# dir_home = '/Users/jerryyin/workspace/notebook/storage-dev' +# # dir_home_user = '/users' +# dir_home_user = '' +# ## user's project dir: homeDir/users/userId/projectId/version/xxx-nb +# file_system_readme = dir_home +'/base/readme' diff --git a/project/main/manager/fileManager.py b/project/main/manager/fileManager.py index cd51b2e..594f726 100644 --- a/project/main/manager/fileManager.py +++ b/project/main/manager/fileManager.py @@ -67,6 +67,13 @@ def getOneNbFileName(dir, fileForm): # ** Except 'system' directory # def getUserHome(home): + dataHome = home + '/数据集' + datasetHome = home + '/system/datasets' + if not os.path.exists(dataHome): + os.makedirs(dataHome) + if os.path.exists(datasetHome): + shell.execute('ln -s ' + datasetHome + '/* ' + dataHome + '/') + all = getAllFiles(home, False) print(all) return all @@ -82,6 +89,21 @@ def getUserHome(home): def getFilesInfoOfPath(dir): if dir == None or (not os.path.exists(dir)) or (not os.path.isdir(dir)): return None + + arr = str(dir).split('/') + if not arr == None and len(arr) >=3 and arr[2].isdigit(): + # dir = "/notebook/storage/userId" + # is user's root /home + dataHome = dir + '/数据集' + datasetHome = dir + '/system/datasets' + if not os.path.exists(datasetHome): + os.makedirs(datasetHome) + + if not os.path.exists(dataHome): + os.makedirs(dataHome) + shell.execute('ln -s ' + datasetHome + '/* ' + dataHome + '/') + + contents = os.listdir(dir) if contents == None: return None @@ -92,62 +114,75 @@ def getFilesInfoOfPath(dir): dirs = [] unknown = [] for i in range(len(contents)): - c = contents[i] - filePath = '' - if str(dir).endswith("/"): - filePath = str(dir) + str(c) - else: - filePath = str(dir) + "/" + str(c) - try: - timeAccess = str(os.path.getatime(filePath)).split(".")[0] - timeModify = str(os.path.getmtime(filePath)).split(".")[0] - timeCreate = str(os.path.getctime(filePath)).split(".")[0] - size = os.path.getsize(filePath) - if os.path.isfile(filePath) or (not str(c).startswith(".") and "." in str(c)): - # is a file - # files.append(c) - # get file's properties - file = { - "name": c, - "locate": filePath, - "size": size, - "timeCreate": timeCreate, - "timeModify": timeModify, - "timeAccess": timeAccess - } - files.append(file) - - elif os.path.isdir(filePath): - # else: - # is a dir - isSystem = "system" in str(c) - - info = getAllFiles(filePath) - print(info) - numDirs = 0 - if len(info) > 1: - numDirs = len(info) - 1 - numFiles = 0 - for i in info: - numFiles += len(i['files']) - - d = { + if not str(contents[i]) == 'system': + c = contents[i] + filePath = '' + if str(dir).endswith("/"): + filePath = str(dir) + str(c) + else: + filePath = str(dir) + "/" + str(c) + try: + timeAccess = str(os.path.getatime(filePath)).split(".")[0] + timeModify = str(os.path.getmtime(filePath)).split(".")[0] + timeCreate = str(os.path.getctime(filePath)).split(".")[0] + size = os.path.getsize(filePath) + if os.path.isfile(filePath) or (not str(c).startswith(".") and "." in str(c)): + # is a file + # files.append(c) + # get file's properties + file = { + "name": c, + "locate": filePath, + "size": size, + "timeCreate": timeCreate, + "timeModify": timeModify, + "timeAccess": timeAccess + } + files.append(file) + + elif os.path.isdir(filePath): + # else: + # is a dir + isSystem = "system" in str(c) + + if filePath.endswith('/数据集'): + # '/notebook/storage/userId/数据集' + filePath = filePath.split('/数据集')[0] + '/system/datasets' + size = getDirSize(filePath) + info = getAllFiles(filePath) + print(info) + numDirs = 0 + if len(info) > 1: + numDirs = len(info) - 1 + numFiles = 0 + for i in info: + numFiles += len(i['files']) + + d = { + "name": c, + "size": size, + "timeCreate": timeCreate, + "timeModify": timeModify, + "timeAccess": timeAccess, + "numDir": numDirs, + "numFiles": numFiles, + "system": isSystem + } + dirs.append(d) + else: + print(str(c) + " is unknown") + u = { + "name": c, + "msg": 'unknown file or directory!' + } + unknown.append(u) + except Exception as e: + sysout.err(TAG, e) + u = { "name": c, - "size": size, - "timeCreate": timeCreate, - "timeModify": timeModify, - "timeAccess": timeAccess, - "numDir": numDirs, - "numFiles": numFiles, - "system": isSystem + "msg": 'unknown file or directory, cause ' + str(e) } - dirs.append(d) - else: - print(str(c) + " is unknown") - unknown.append(c) - except Exception as e: - sysout.err(TAG, e) - unknown.append(c) + unknown.append(u) result = { 'files': files, 'dirs': dirs, @@ -160,6 +195,15 @@ def getFilesInfoOfPath(dir): # return content +# +# +# +def getDirSize(dir): + size = 0 + for root, dirs, files in os.walk(dir): + size += sum([os.path.getsize(os.path.join(root, name)) for name in files]) + return size + # # move file to dir # @@ -328,10 +372,18 @@ def createProject(path, projectId, projectName, type, filePath): def deleteFile(path): try: if os.path.exists(path): + if '(' in path: + path = str(path).replace('(', '\(') + if ')' in path: + path = str(path).replace(')', '\)') + d = shell.execute('rm -rf ' + path) print("d ======== ") print(d) - return (1, 'Delete Sccess!') + if d==0: + return (1, 'Delete Sccess!') + else: + return (0, 'Delete Failed, Please check file format!') else: return (2, 'File not exists!') except Exception as e: diff --git a/project/main/manager/projectManager.py b/project/main/manager/projectManager.py index 6241f8a..7e40ef8 100644 --- a/project/main/manager/projectManager.py +++ b/project/main/manager/projectManager.py @@ -32,14 +32,20 @@ def checkVersion(userId, projectId, v): # projectType -- 'PYTHON3' | 'PYTHON2' | 'R' # def createPreProject(userId, projectId, projectName, projectType): + userHome = '' pjHome = '' + dirData = '' if config.dir_home_user != "": - pjHome = config.dir_home + "/" + config.dir_home_user + '/' + str(userId) + '/system' + userHome = config.dir_home + "/" + config.dir_home_user + '/' + str(userId) else: - pjHome = config.dir_home + '/' + str(userId) + '/system' + userHome = config.dir_home + '/' + str(userId) + pjHome = userHome + '/system' + dirData = userHome + '/数据集' if not os.path.exists(pjHome): os.makedirs(pjHome) shell.execute('cp ' + config.file_system_readme + ' ' + pjHome + '/') + if not os.path.exists(dirData): + os.makedirs(dirData) #create directory for users's dataSets pathDsets = config.dir_home + "/" + config.dir_home_user + '/' + str(userId) + '/system/datasets' From 30059560ce9e730c450c1b8da7dcacccc714762e Mon Sep 17 00:00:00 2001 From: JerryYin Date: Tue, 20 Nov 2018 13:46:09 +0800 Subject: [PATCH 10/42] fix dataset dir bug --- .idea/workspace.xml | 106 +++++++++---------------- project/main/manager/projectManager.py | 8 +- 2 files changed, 43 insertions(+), 71 deletions(-) diff --git a/.idea/workspace.xml b/.idea/workspace.xml index d588bd7..8839137 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -3,8 +3,6 @@ - - - + - - + + @@ -107,8 +105,8 @@ - - + + @@ -123,8 +121,8 @@ - - + + @@ -132,17 +130,13 @@ - + - - + + - - - - @@ -161,7 +155,6 @@ - .system shell startVm sys @@ -174,7 +167,6 @@ resp_err_version_invlid custom.css executingNb - createProject unknown split bindDataWithProject @@ -188,9 +180,11 @@ dir_home initProject getFilesInfoOfPath - createPreProject dir dele + createPreProject + createDir + createProject @@ -224,9 +218,9 @@ @@ -237,9 +231,8 @@ DEFINITION_ORDER - @@ -268,19 +261,6 @@ - - - - - - - - - - - - - @@ -413,8 +499,8 @@ - + @@ -443,10 +529,9 @@ - - + - + @@ -455,7 +540,7 @@ - + @@ -469,30 +554,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -680,13 +734,6 @@ - - - - - - - @@ -708,23 +755,15 @@ - - - - - - - - + - + + + - - - @@ -774,20 +813,6 @@ - - - - - - - - - - - - - - @@ -795,13 +820,6 @@ - - - - - - - @@ -844,23 +862,61 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - - - - + + + + - + @@ -868,22 +924,39 @@ - + - - + + - - - + + + + + + + + - - + + + + + + + + + + + + + + diff --git a/bin/automount b/bin/automount index fc92e5c..ca6f784 100644 --- a/bin/automount +++ b/bin/automount @@ -10,8 +10,11 @@ set timeout 20 ## dev server #spawn sshfs root@60.12.136.60:/sshfs/notebookserver /root/notebook/storage +#spawn sshfs -o reconnect -o ssh_command "sshpass -p abcd#12345 ssh" root@60.12.136.61:/mnt /notebook/storage -o ConnectTimeout=30 -o StrictHostKeyChecking=no + ## release server -spawn sshfs root@60.12.136.60:/sshfs/notebookserver /notebook/storage +#spawn sshfs root@60.12.136.60:/sshfs /notebook/storage +spawn sshfs -o reconnect -o ssh_command "sshpass -p fd324;1 ssh" root@60.12.136.60:/sshfs /notebook/storage -o ConnectTimeout=30 -o StrictHostKeyChecking=no expect "*password:" send "fd324;1\r" # expect "*#" diff --git a/project/main/config/__pycache__/config.cpython-36.pyc b/project/main/config/__pycache__/config.cpython-36.pyc index fe17b25940bbf38047174a85b466cf3cd2d0936f..531ff2e166669855e46c934514c78de9eaefae1d 100644 GIT binary patch delta 489 zcmZ|Ky-&hG6aesRdxw>_LR(s(fDjRr4z}ULj~HVdOk50$lXdWeYmk^w1C3D@7iSZ4 ze}IdllPqrj1O6AfI=ZmvtAimL?&aR^UGC*Rm@<~jbw!E4Ha-qF69~PbiNQm?4YU6^ zEHB@9aT@m|DtWRmQ8|q~g(}`0)N@cLzyz2Clb{N!payE74(ebEOo0YyfF@{yX_}^) zTjXVEmRewz=4c+YXo1dyIZBHA$Sw`vy*;_QR&%PWPTi@kG&&DOggVa)2n|YSVR+Hs za9!$OIyC6EuR5)k)9Sk2AoNdz;C!I|mTo@`dM9mvbjJ}srot!3O?J##Y$P0UIvd~$ zTgS2yZS?*mjUC_;)9@0z$I_IJ7i_U>==j33Q#uO7nla(@rk!V7Bw$Zu2NPJcPhv0X n(Y2{48BvP*{=Z^3R<=#d{RGB5=EVxuL=J+V$bx8!f+&0gK;vG; delta 503 zcmZ|LJx{_w7zgkxw7j;ImzF{SL5W68prS+*2IHVBF)l-?Z@7ljLBE2% zJY1Yzde1q9eR->D96o_gM1pPd1@_F(u^T@8&y9Ot=aMjCmVNTUm5BjAZ2~6023y40 z+HEvX$jJR|4+P?<2H7a(*n(gQGztwH2^;K9m@h@3p^6)-8%a0Z+kc-}@fGD7&$(Oi N@*dt32(W#h{tfs!XaoQN diff --git a/project/main/config/config.py b/project/main/config/config.py index ccbf2ed..42e37b4 100644 --- a/project/main/config/config.py +++ b/project/main/config/config.py @@ -89,7 +89,9 @@ #cmd #sshfs for storage on StorageServer -#cmd_sshfs_mount_storage = 'sshfs -C -o reconnect user@hostname:remote_dir local_dir' +# cmd_sshfs_mount_storage = 'sshfs -o reconnect user@hostname:remote_dir local_dir' +# +#cmd = 'sshfs -o reconnect -o ssh_command "sshpass -p fd324;1 ssh" root@60.12.136.60:/sshfs /sshfs -o ConnectTimeout=30 -o StrictHostKeyChecking=no' # sshfs_pwd = 'fd324;1' diff --git a/project/main/main.py b/project/main/main.py index 2a2b66c..85f1add 100644 --- a/project/main/main.py +++ b/project/main/main.py @@ -3,7 +3,7 @@ from base import sysout from config import config, string from server import httpServer - +from manager import fileManager @@ -18,3 +18,5 @@ print(warning) sysout.info("", TAG + " is starting...") httpServer.run() + # fileManager.getFileNumber('/Users/jerryyin/workspace/notebook', True) + # print(fileManager.getDirNumber('/Users/jerryyin/workspace/notebook', True)) diff --git a/project/main/manager/__pycache__/fileManager.cpython-36.pyc b/project/main/manager/__pycache__/fileManager.cpython-36.pyc index 448fd53086b57cf82b7ebcbf7cf509bfcd57cbaf..a01f80f30c82d0328614fe0000a770bd14baa22e 100644 GIT binary patch delta 3907 zcmb7HYiwLc6`r~G?(W@(ch~klY$tZE?L54BIcb|Tk2(z`3L)gxIH+kJEt{Qdd)NDL z=3diybvH|E7lIJT)ompt@~0pn@d}{);2*#EjZjq~wBkplN~IP*r~o11Kk%Kon|F)E z+H+>+%*>fH-<&z;+)wuWVru-^o}Sq4U;g9D*t3j%#2)$SXdgv!e*5*X z?FDP>)m>wr^?C`+W<-r$unG}rVm}!r5+gz?bb6*2ub-FgM#*-ZV>Qu<&D!p{MfZz~ zGc%n~soHStLP#cPW~$VvyTx+-*j%|Pj+bReFJXRYp|N1sJ9>SwCX*ypk(3e*Q76h( z`|M)vx-IvD<@A9tp5q2js1oBQ{^WQY?9YZrV8mq}Tjn?}6Kq~{MqFOeDgbDiCzu=) zY??g*-;u}QV+0;&u2Bg+fD!O17$X6U?a#weJdLA?Gs|+_{VX`;MyM-?iHv}Bp_7}4 zVG$W6GDhSvu=cP{0%!r%0IZ{!;p9LV;CTp_$-97U@z3c0;V=1Tj7{wc%qnr3i=Tl) zrmsxvIvT)?!>C|hfpNL40`xq-%$?I{>(l}^0T(#Q(}WZ7gDDCl&pfOap}fN14MR$J zS!?NT#M9M{=+kXhh@4#{gPQ#K-CuqD-Mec)_+~Rb=`1;}U7Hj|x9Hfe0~4)VyBw>j zPj0u?`Q(X$`6PaEhzM*lV&Ak&i>@s<5Nk6?!RQ!sjhfvtN#IFp#k->UN&h#-<^win zbq04YvZ2FtpO^MWI^`H;K| zu0F^ruotuyGI*#88K4oM2Sl7(Xqy2x4>5J_DsEtf(Mn8&Jsqe_`m?MRLJ6ZpR<)al+(a;AnIC7X+O>(6>0-TS4nmTM;<bE(n=rY+!tey#`kkvfouJWZfT|=EH>BW|@7-+>wtwGj` zcSllBjU>7w>8D0|l&oIjw~}qu>7&jncRmQVt@P6d2?j0yF0chDI3K2-7xiLZ((CaO zUR)&Zgj*>uRLOWLGRXR$lq_Az(&Udt^!~~gl)$-x<*FvPqaC6)9YC~U-+(3?XyfKo z-RtJPUJ@zHRv#cnTlCBuSp2p7y#dzh!)X7_5wG_+yOwL2UO&c6ktFb+XRZEK?MCHg z28FxodVOBMmlmlzW-H@`p>&3JXxBHp`9M#R4s5}TQdGL5wX$9W)3P+}Wp?9JgSi9q zu~l6TaMsFk*4*t5R)*XSUT#@~gN%xP$jQ-FKE3)CxKDvqw{n*ti3(H;d13ELL}c#h zt-P0qCxwX>vS(!hPh^ce$r^=;|L=WE@-c$kf=HVr0O;vww~T_Yv7o+|)->vd}eVZ&-jOO&NuavO4Kw4kYl9lhqvHb*~q-cGAjTy$)!piA0L z_JedH-RXsme%j0(l;l`DA!nglb~}0%(JNARM0vLC$_GA3ZdVJdWyggMoscZnXKnwe znf0%l`}$RsgtMM5H}5+s|NG|1xq%>3>P0E#PTGDKNiivK+I81?sFHkv26(x{OC6IA zd%oz-$pf@R;h~V(=l{zbOB^ERpqjquk4ASKT0+}-2ZZs>+|ZJIOz(O;jl5pq^HhM< z3GN*IM~!6CH; zeB{k%=$Vl#f!%Mus(%}43(^zrYTsfr@R&khffqcEarF-PjWkS+(jnulG7cr3g1*Xw zNIYGXLxFJ?A?w;ui>r661+V!w7B9nnzOCIr#-l=K!FVKod;#UM}pW(*SY(JYNJ%FI!DmBKNrTNQl~6! zqdSW=Y)&arV8x_)aI+NB3tV0z24z6<6v!ti4lQTuDn_{CQnGXJ578n|(nD+NwXY*8 zTJoU(VtoJL6|i9M>d{;c+6Pg|SN-qBhxSn5QgKq*sQe0b>2l<0A`uYiJ49`z<3<0^ z@s0eS{=ee|{@T6q#2Sy&mQqRF>*tfsD@UmRA`z0^2}`?Ptl5EN5?YAh)Uwpkh2wU( zBhS%P^^8?Ad6Bx0r87(x!HaYMqvR=G^mnIz#0&oWsbl=0|6cEazdJp4nBw*QGtLBU zrO_<44fRByI3#9w5A>b%hT)??=Rv#=tX<l+z9^FOw&-ZT0w9==8^3;y=Z zHXQdt<`QrCKg*2q?f&mGQ(G2j@CHc7aF-Tr`4aVAA~hm)A`O2}c9hrs3)vmK<=@H< zwtoork{i4TdUUc*3GaMZZgo`}TrmV59TGD}i7)s^9`JQUIq=<3B`iYpO+hJ+8$lX} zh=Mc@4-|2OiwN?IIgvyFNKLc4A=1-`H}wg`8*{!LlXnnq5MY`K5fMF- ziEjTy5mzR5TIZ`a(#Nb-nzKvuK~$J&$Xd}Im3#fkT-N_`Zo_qDH{XD``}Xuk(B6wm zzDe!qBfcU=TonN#_*AMEOZFpnM!9Bi>w55^G)Rn#{?BtGg?H%cDXVAjk_q@w@~Ub0 zPx8L-uUc3mo?P+&lDiz0hcOWft@x+%L*chb%wfNh-?Z%~B!*rK`2$K)4Twu>@;1%q z7YqCZ{SWv*%}-5@)3Pl@CWufTtDLiPA@Viq7KzLesS;@sSs`-U9~#)oC;TrA^bf>R Xv0gKSzlNDG^X5LY#~d}cn%VyX__;wv delta 2414 zcmZWr&2Jk;6rb5$d)J@wSM0=%Q#T1s-L#_65>VT;X`7}|i$Y0Co2E3y@@x`2{zzvx zZDMwtgAp9K1kDL4;($0*6(FJh0p)_k6^R4d6Q~kxH~PO3m{&6_uGX5QDl z_p`qmd3vP(Xe1IWe(~Flp9cwfOm=-#;74It4?gLiSnp#&7W#nB23eRzJ|MFp7G)hU zhgpopVUDl_OTrvwDb@*d2TQXI%rVx*x?zs9EZYNfg7vT*%t_YE_QITEdA1McPDZCm zp??byOKs}V7Nx{N<;vuELE(GgfahE0<+@!^JZ(`t)&~xq$E$uIga)ZWc^`1k^`~GF zhavUPDB`p+v`#9TLt4}Y{fsOr{1hY8#P*>zMqxc!hV=q*6&NZEZG*1W0j(CHvP1(9lDzp-C z^cI+}1eu0TfLddFC-lNjV0$P0!cI`?ilE#LNj{4F2DMJMovCzy{&gCL2>2}1+QKD2 zt{lIkJ0U0N1RR6;HvMkcQ7SPfj4x2XFO{inW&A1g@7#1Fc3hf^+6gC$XBl-nAXC6I zeF-whKi^q(5w`<&W9zHEPIQdiOhXD|u<2t)HzeI@U?JutpaAn&Or^AUk5f<1q z+_Gs6uO1ds%Qc)qg5%G}edWy?lJt(tYu$BXp^lVc-z zhFNFU-Lk#t84IRew97RUI%~OFwnZ>9Y|3Wfd+}u8{A$Tuw#$t=$6oSOlXI>ES4i=& zR8?x!ZL@A$(pR1Y89$60wCvH6XUvzY=H;Tj$PeO<^jyX;D87yK1rDLG8;JNlGPHjR z7>ffTbbxBIhhXC4!i#5*y^1WNg{PaVW@*JX`Ee9s!2ARf8L-N; z$Ya#}jQF7^88m8*JLWh9cwd}J6zIgmrNkBup?!|(w-j+JWnDjp>mx|ekmu)Sy;$2e zf!YhYr&_$^sm!uHYVi>~Dqmc_=owr_pOX&cE&+KBr&dP#&bIUzbfqyr+GK1YYd zT>AKlx8T4|z?a*+4ESy2HCY5QVtxSxGFmvjD>A)STr$VX{4XN@OlRp782JngJjk4q z2D$|8%8gF%GIC{?k#~^ywbfy{y=X}$5_6aemAF~7WfH0QBfV$+DqMIDvG@Ww@ck?V zUq%tc5&D<{@CQI%Q4m}!Y70QsQJ|gF$)>+*myo=RqZIR*k5 zGXrLASVE+Naq}a#TG1GF2%;0j?_&yLQ~L)5ATj`;5}XFCsviKX@-GGX93T~7Q!~tT zW;l?wdB_p~lX6d)A$1t+D56kkvZ?c0{xF}#K`!PTzvK*(uVquPcu zga+Acy}1c|j0CI+bbA<{!$3Gt3>u@nA-Zz20Z*-23t*}t-p^(I%Xsy+%=Fi}@V+Z} z9ovXVWC2tGHBp7zASuu*g|zr3H}VGlfsnuQ{48=4NU+)X6cVgWegg@<7;hlK`#g$f shK-Z823x6`r?6x#gaHE_m0v|ujbt!kq>PAh)yNpH!JIZi#(pFGFZgx)?EnA( diff --git a/project/main/manager/__pycache__/projectManager.cpython-36.pyc b/project/main/manager/__pycache__/projectManager.cpython-36.pyc index 08dffa6cbe9028a8ade764573d5f5040a5b486bb..316f9957b29892dbd5fa27537392ccfa4183e5d1 100644 GIT binary patch delta 512 zcmX|7&r1S96rQ)UvpYL#g%E;RcoDJi2PB=kRZw&ZyojBQ@z9nQx?W_&U_rV>!P2SD z9i>=^4nf!G-zbDG9y>;FSCQfIzW2WGeSFM&{;eQ3VlnaMOpeN>XHles2qDD4+Q#Zs zfe;CO9gATZ4zWSPDQ%Y#X;Mp&V3QyrqKdeP`+6%rXXiVmONtzGbeq|RVlH$p5t{)5 z;&9uNI$jBmW-LJz^ImCgV9w+w%3ne`A|PmHqM0#c4P+-Fi)d!X6)L2KPR2A|$OQ3& z(6d-lgcyR@GJ=jg2(BhhnTVyC{0ntUDg-Qg1zW0$tYTh;M~

jAm delta 364 zcmYk2Jxc>Y5Qb-NcklK#ISm$fei8|!2*E_~7l`1S?IfOCv=(k%8m> zjkT`CpIVUmajGUFbs6~IIhv;6G(}Q+SZ2GBD6$n(by1EtIH(aPE^_Ds$6T0;k+d^r z*4A`PH_CE54-lN}fmV(@c@J6@f6v?gi@#&+pG7kYFU;=wY2SZ~?S%$?0g$7@Mv!8a v`Z8NBdvO$AGu$jb<8gp}UsNY}TRbEy0`H3P+%dyR(M~1Cb=g!Msnq@t#>-EM diff --git a/project/main/manager/fileManager.py b/project/main/manager/fileManager.py index 594f726..f84fa62 100644 --- a/project/main/manager/fileManager.py +++ b/project/main/manager/fileManager.py @@ -27,7 +27,7 @@ def createDir(dir): # 'files': ['.DS_Store', 'migrated', 'jupyter_notebook_config.py', 'jupyter_notebook_config.json2'] //all files # }, ... # ] -# +# ls -l|grep def getAllFiles(dir, containSystem=True): if dir == None or (not os.path.exists(dir)): return None @@ -42,6 +42,36 @@ def getAllFiles(dir, containSystem=True): return result +# +# get files number of one dir +# By Linux CMD +# +def getFileNumber(dir, containChildDir=False): + if dir == None or (not os.path.exists(dir)): + return None + cmd = 'ls -l '+ dir + ' |grep "^-"| wc -l' + if containChildDir: + cmd = 'ls -lR '+ dir + ' |grep "^-"| wc -l' + popen = shell.SubProcessCmd(cmd) + num = popen.getOutBuff() + popen.close() + return num + +# +# get dir number of one dir +# By Linux CMD +# +def getDirNumber(dir, containChildDir=False): + if dir == None or (not os.path.exists(dir)): + return None + cmd = 'ls -l '+ dir + ' |grep "^d"| wc -l' + if containChildDir: + cmd = 'ls -lR '+ dir + ' |grep "^d"| wc -l' + popen = shell.SubProcessCmd(cmd) + num = popen.getOutBuff() + popen.close() + return num + # # get only one file name of the dir # --fileForm: .ipynb / .html / .py @@ -91,7 +121,7 @@ def getFilesInfoOfPath(dir): return None arr = str(dir).split('/') - if not arr == None and len(arr) >=3 and arr[2].isdigit(): + if not arr == None and len(arr) >= 3 and arr[2].isdigit(): # dir = "/notebook/storage/userId" # is user's root /home dataHome = dir + '/数据集' @@ -103,7 +133,6 @@ def getFilesInfoOfPath(dir): os.makedirs(dataHome) shell.execute('ln -s ' + datasetHome + '/* ' + dataHome + '/') - contents = os.listdir(dir) if contents == None: return None @@ -125,11 +154,12 @@ def getFilesInfoOfPath(dir): timeAccess = str(os.path.getatime(filePath)).split(".")[0] timeModify = str(os.path.getmtime(filePath)).split(".")[0] timeCreate = str(os.path.getctime(filePath)).split(".")[0] - size = os.path.getsize(filePath) + size = 0 if os.path.isfile(filePath) or (not str(c).startswith(".") and "." in str(c)): # is a file # files.append(c) # get file's properties + size = os.path.getsize(filePath) file = { "name": c, "locate": filePath, @@ -149,15 +179,16 @@ def getFilesInfoOfPath(dir): # '/notebook/storage/userId/数据集' filePath = filePath.split('/数据集')[0] + '/system/datasets' size = getDirSize(filePath) - info = getAllFiles(filePath) - print(info) - numDirs = 0 - if len(info) > 1: - numDirs = len(info) - 1 - numFiles = 0 - for i in info: - numFiles += len(i['files']) - + # info = getAllFiles(filePath) + # print(info) + # numDirs = 0 + # if len(info) > 1: + # numDirs = len(info) - 1 + # numFiles = 0 + # for i in info: + # numFiles += len(i['files']) + numDirs = getDirNumber(filePath, True) + numFiles = getFileNumber(filePath, True) d = { "name": c, "size": size, @@ -310,22 +341,6 @@ def createFile(file): except Exception as e: return (0, 'Create file failed, ' + str(e)) - -# -# get the directory number int the 'dir' -# linux : -# get dir number of the dir -# #ls -l |grep "^d"|wc -l -# -# get file num of the dir -# #ls -l |grep "^-"|wc -l -# -def getDirNumber(dir): - cmd = 'ls -l ' + str(dir) + ' |grep "^d"|wc -l' - n = os.popen(cmd).read() - return n - - # # copy init common project to user's dir for preview # path -- absolute path of the file @@ -380,7 +395,7 @@ def deleteFile(path): d = shell.execute('rm -rf ' + path) print("d ======== ") print(d) - if d==0: + if d == 0: return (1, 'Delete Sccess!') else: return (0, 'Delete Failed, Please check file format!') @@ -394,12 +409,11 @@ def deleteFiles(paths): for p in paths: code, msg = deleteFile(p) result.append({ - 'code' : code, + 'code': code, 'msg': msg }) return result - # for test # if __name__ == '__main__': -# getAllFiles('/Users/jerryyin/notebook') +# getFileNumber('/Users/jerryyin/notebook') diff --git a/project/main/server/__pycache__/httpServer.cpython-36.pyc b/project/main/server/__pycache__/httpServer.cpython-36.pyc index b0da3d0dc763395ebf9204693bf2bb43798d36a9..ed1b3a297771b1e301316ae726070394990e7c4e 100644 GIT binary patch delta 18 acmdmLy494^n3tDp*^1B678^O&NC5ymrUr=s delta 18 acmdmLy494^n3tDp|H4<%mK!w2n3tDp*XqyFJ2!IfXL4j>U|?_t;$jUTQK3}Bkj0S2n8no0kj320P|H}u zSj1Grn8lL9*vuHrP{l+?-M%wO5$ NfF>0wOcr993;-&*93cPz delta 90 zcmdnRxs;RBn3tF9<riPXEI@AU|?_t;$kHrQ6W>qkj0S2n8no0kj320P|H}u qSj1Gr7|c+}6wZ*s5X_*-=%>jtnUh(1vJUfCHd&yaBKgVcERzB3 Date: Wed, 21 Nov 2018 09:58:35 +0800 Subject: [PATCH 12/42] done delete file bugs for illegal character file name --- .idea/workspace.xml | 171 ++++++++++++++-------------- project/main/manager/fileManager.py | 54 +++++++-- 2 files changed, 128 insertions(+), 97 deletions(-) diff --git a/.idea/workspace.xml b/.idea/workspace.xml index 2545d43..c279ccd 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -3,12 +3,7 @@ - - - - - + + + - - - - - - - - - - - + + @@ -111,7 +100,7 @@ - + @@ -128,8 +117,8 @@ - - + + @@ -144,8 +133,8 @@ - - + + @@ -156,7 +145,7 @@ - + @@ -168,8 +157,8 @@ - - + + @@ -180,17 +169,24 @@ - - + + - - + + + + + + + + + @@ -202,7 +198,6 @@ - resp_err_version_invlid custom.css executingNb unknown @@ -232,6 +227,7 @@ getFileNumber getDirNumber size + syso @@ -319,6 +315,13 @@ + + + + + + + @@ -531,9 +534,9 @@ - + - + @@ -748,13 +751,6 @@ - - - - - - - @@ -855,13 +851,6 @@ - - - - - - - @@ -869,13 +858,6 @@ - - - - - - - @@ -890,20 +872,38 @@ - + - - - - - + + + + + + + + + + + + + + + + + + + + + + + - - + + @@ -914,48 +914,49 @@ - + - - + + - + - + - - + + - + - - + + + + + - + - - + + - - + + - - diff --git a/project/main/manager/fileManager.py b/project/main/manager/fileManager.py index f84fa62..49d15d5 100644 --- a/project/main/manager/fileManager.py +++ b/project/main/manager/fileManager.py @@ -1,6 +1,8 @@ # charst=utf-8 import os +import datetime + from system import shell from base import sysout from config import config @@ -49,9 +51,9 @@ def getAllFiles(dir, containSystem=True): def getFileNumber(dir, containChildDir=False): if dir == None or (not os.path.exists(dir)): return None - cmd = 'ls -l '+ dir + ' |grep "^-"| wc -l' + cmd = 'ls -l ' + dir + ' |grep "^-"| wc -l' if containChildDir: - cmd = 'ls -lR '+ dir + ' |grep "^-"| wc -l' + cmd = 'ls -lR ' + dir + ' |grep "^-"| wc -l' popen = shell.SubProcessCmd(cmd) num = popen.getOutBuff() popen.close() @@ -64,9 +66,9 @@ def getFileNumber(dir, containChildDir=False): def getDirNumber(dir, containChildDir=False): if dir == None or (not os.path.exists(dir)): return None - cmd = 'ls -l '+ dir + ' |grep "^d"| wc -l' + cmd = 'ls -l ' + dir + ' |grep "^d"| wc -l' if containChildDir: - cmd = 'ls -lR '+ dir + ' |grep "^d"| wc -l' + cmd = 'ls -lR ' + dir + ' |grep "^d"| wc -l' popen = shell.SubProcessCmd(cmd) num = popen.getOutBuff() popen.close() @@ -235,6 +237,7 @@ def getDirSize(dir): size += sum([os.path.getsize(os.path.join(root, name)) for name in files]) return size + # # move file to dir # @@ -341,6 +344,7 @@ def createFile(file): except Exception as e: return (0, 'Create file failed, ' + str(e)) + # # copy init common project to user's dir for preview # path -- absolute path of the file @@ -387,14 +391,15 @@ def createProject(path, projectId, projectName, type, filePath): def deleteFile(path): try: if os.path.exists(path): - if '(' in path: - path = str(path).replace('(', '\(') - if ')' in path: - path = str(path).replace(')', '\)') - - d = shell.execute('rm -rf ' + path) - print("d ======== ") - print(d) + # if '(' in path: + # path = str(path).replace('(', '\(') + # if ')' in path: + # path = str(path).replace(')', '\)') + + # path = checkFileNameForm(path) + # if path == None: + # return (0, 'Illegal file name can not be delete, pelase rename file name and try again!') + d = shell.execute('rm -rf "' + path + '"') if d == 0: return (1, 'Delete Sccess!') else: @@ -414,6 +419,31 @@ def deleteFiles(paths): }) return result + +# def checkFileNameForm(path): +# arr = str(path).split("/") +# fileName = arr[len(arr) - 1] +# if not fileName == None: +# if '&' in str(path) or '>' in str(path) or '<' in str(path) or '^' in str(path) or '?' in str( +# path) or "'" in str(path) or '"' in str(path) or ':' in str(path): +# # inlligal character +# newPath = path.replace(fileName, "serverdeletefile" + str(datetime.datetime.now()) + ".txt") +# r = rename(path, newPath) +# if r['status'] == 1: +# # rename OK +# return newPath +# else: +# return None +# # try: +# # shell.execute('mv ' + path + ' ' + newPath) +# # return newPath +# # except: +# # return None +# else: +# return path +# else: +# return None + # for test # if __name__ == '__main__': # getFileNumber('/Users/jerryyin/notebook') From 72d3de01db0575b6efe05befdd20746a7dbc3765 Mon Sep 17 00:00:00 2001 From: JerryYin Date: Wed, 21 Nov 2018 20:54:41 +0800 Subject: [PATCH 13/42] done get threading & asyncio for [getFilesInfoOfPath]; added log in sysoout.util & save all the logs to file [NotebookServer.log] --- .idea/workspace.xml | 417 +++++++++--------- project/main/base/sysout.py | 26 +- .../config/__pycache__/config.cpython-36.pyc | Bin 1658 -> 1683 bytes project/main/main.py | 16 +- .../__pycache__/fileManager.cpython-36.pyc | Bin 7853 -> 7656 bytes project/main/manager/fileManager.py | 82 +++- .../__pycache__/httpServer.cpython-36.pyc | Bin 6837 -> 7652 bytes project/main/server/httpServer.py | 45 +- project/main/system/shell.py | 50 ++- project/test/asyncioTest.py | 54 +++ 10 files changed, 434 insertions(+), 256 deletions(-) create mode 100644 project/test/asyncioTest.py diff --git a/.idea/workspace.xml b/.idea/workspace.xml index c279ccd..3b57cce 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -2,8 +2,13 @@ + + + + + + + + + @@ -382,7 +366,7 @@ - + - + - + - + - + + - @@ -533,6 +517,7 @@ + @@ -540,13 +525,13 @@ - + - - + + - + @@ -555,7 +540,7 @@ - + @@ -591,62 +576,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -788,13 +717,6 @@ - - - - - - - @@ -858,13 +780,6 @@ - - - - - - - @@ -872,13 +787,6 @@ - - - - - - - @@ -893,6 +801,37 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -900,63 +839,117 @@ - + - - + + - - - - - + + + - + - - - - - + + - + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - - + + - + + + + + + + + + + + + + + - - + + + + + + + + + + + + - - + + + diff --git a/project/main/base/sysout.py b/project/main/base/sysout.py index 783cc11..9a618f2 100644 --- a/project/main/base/sysout.py +++ b/project/main/base/sysout.py @@ -1,28 +1,48 @@ +#coding:utf-8 + +import logging +logging.basicConfig(level=logging.DEBUG, + format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s', + datefmt='%Y/%m/%d %H:%M:%S', + filename='NotebookServer.log', + filemode='w') +logger = logging.getLogger("NBLogger") + + isDebug = True def log(tag, msg): if isDebug: try: - print("LOG: " + tag + " | " + str(msg)) + log = "nbLOG: " + tag + " | " + str(msg) + print(log) + logging.log(log) except Exception as e: print('Exception:') print(e) + logging.error(str(e)) def err(tag, err): if isDebug: try: - print("ERROR: " + tag + " | "+ str(err)) + log = "nbERROR: " + tag + " | "+ str(err) + print(log) + logging.debug(log) except Exception as e: print('Exception:') print(e) + logging.error(str(e)) def info(tag, msg): if isDebug: try: - print("INFO: " + tag + " | "+ str(msg)) + log = "nbINFO: " + tag + " | "+ str(msg) + print(log) + logging.info(log) except Exception as e: print('Exception:') print(e) + logging.error(str(e)) diff --git a/project/main/config/__pycache__/config.cpython-36.pyc b/project/main/config/__pycache__/config.cpython-36.pyc index 531ff2e166669855e46c934514c78de9eaefae1d..2353e62e2e4572d85a4359f08c0a0270fb64fc60 100644 GIT binary patch delta 547 zcmaKoy-UMD7{)J6qV^+AKhj34wQ3P;p^fzuoE#jSiXaZg#p)eei%q!%#V!iE34$Dg z;_83kBI19clbbraI=Sc*aS#RXxaar0$NSuI_Z|^ieq7T+Cy#F{_c|dj4b((~ROu|Aecqo~~4%4t5p%GhS5vrk%(x|P&>+mr!2FAfSm;e)C5=?>yXn-j& z1*X9?Xo4o#O}lB&5wUw{hW3IPn(Cujm|ogXb6_7GSR{qP(+7JYSQ(oxk78EpS2;rJ zH$#N9J2ySAHDg)t(oDN-ExC-l))wP@Z?E38b{*buTeWp&H64$wIZnfJJ%`seSc$To z>$ao@q=Yz-SEYUNBxfZ1)t*$-a+2%7NKLRJ)2N98dT;zFGh&+14aih>)y fqph1PlYizz8q`RDmim zN}?oojf@zHlLRnMk|YI8kXbSZOcFl5hqO%p-PjR}OC`Nn)606PT4_I|5o$lrBh!RIk>dj7}V>#xLWu5foUt8!pmfdKXgByltJu(bAtkXj#K?mGM z`qG^_41ze8VHpn|tH|0R+QWIejKzu@v!_IX+H0U?I`yAN>y1;BLb2=qCSs_J7wA3q z=caO_>nQAn@OIUW-O>&&(l6|-{oA~|?>66!->_BibuC5Lc#A&qo0x|?{p7V=2#oDc kW=C9c`~D0-w>()*#ms19%wkr^V?|(KvlGe*s=x`{H-0{E%d(XM& zoO|~D(eFoAcZ5R5J0Jgg{OYhIU6bzmD9HEXus=g!5`&sl>_{jA&gGh1SBeUdPN}JN zU(x4rwWigzqULd5&0p7xI&!TTV1A}wmWl=&U5fUKe^DhTTQ#o8{2P#(Nsv~CoLr^L>2{MX z**g_OJNuI@NK5}cK247Wk6lbop(z^2B-iI^uAjx%lxE6JP2&2bR7gzlL2(Wr$x)%1 zZ$nP66 z6cs;2;6{*wf}eIWss7W2+s4;oa1Hhu;;3}{(7btwBvNaLGej;03F`Q zPC-NL&Gq4XozIPKUZTZigtqFOUoP>dXl@hK2<9P9G|1 z1jjfIn^JQAVXyCH7aNstca)0Cz?HDyWNO_exWJUE%&#b#YGnZg1to?4Kp@0#*Oc}i-?Q{9~=3{9-)yxw~+U+C?-4Sk(wIP+Tf(qXSU z7x5~0s%Yv}NjxRB3j&(X)o5O2#nw}kXl>sKA43^BFr!#Q000-!o^={SH z8CKCL*GuBN{OG9n7coHQ?jx8apho2<31$gS5mX7NGW-$&S>_g9V7I=zzQoSdN_(hO n_ErQP@ZSM%fnQlw^tcf-!bVt6;PS=wa9@1m_7Crd~xH6vo delta 2141 zcmZuy-EUMy6u;-*yZ7F``_eT-M57_ngclS40DLn(m`zOlhzZ0OqdwsSo^#g{qU`Sc&Tr0~nKLuz zoY{wCUyQHp4Ft@!yMJEj-787INY8u} zJrz&R<8n_~tLQl$xt24S&Wu$l=VjZO$$YDjGua>uuplxYo6WKiYeVX17Hdbd01LAS z${>rf7|IZfvjoaEmSicE7VBW0DBD>VOQQ_4Zk9nAVLhxDWt44WeJEqBpADdlfBEng z0RQHY+zrsZIiVZ`*ex3B{woa6WlN6K06=6?Nv@@BD9S~Jbbg8VGHD9W?MQJvbzG{; z4)DVaN4NSoTc+8zTJ#XBD~?js8d6V7tIO93q<6dbDT}xm< z3{>#D%>Wbo2JrsPdmh~Bp!in%!NdE|PTbcEp#%~O+fi(_J&M?6{A!Kbep_?Y1~lN9 zv@Ty76n)-lcvm>y$x&M>LeX+e+*Huz?Znri_MNRd-UbxI%twb@2mS~1wR9zodcrWf zwtw%=QQSE)|AcwBEb48JSqw0Ax-Rn_;*^nTK40ebQ{fDOXt9iRtSk5=KLT$cK&Nt1PJ*s5DyPE!&neN`Puk~rYwDHRQXXS=M6Aa; zU;#aTgx)6Nhh1+8aVOq&q|ox>UTd+%U5_Q=p4QsWcbjtEiYM_joOT_QH%Aj0&@Pca zE3PEBqnowl3@nPDlD(L!YHGZHi3YDBG*$cNQl7s?eVd>{P$gIt^Qj)FinY|xN*xt* zv^n}vK-;iUIBItgZrbicBGZe?tMrzZG(KRvFKkg|z~qKp$Nv#mfB30h!3PHQwx_5u z552ee#(=FejY`|r9enYyfPv{Hyxj#B#99nZNmZ4#O=0C%2eI;vlF2u)?y=IB{rDFY z5ZxWgVQca+3@*zT2OmQgMRr-|mCP`o&n){`X8hkzrj^4ZVzOg#n|s%{@F-hRf2YNF zqu}on_ljFqWT>h0{8D*#F8@sUsfOKb%qp#nOOoQO@OP%OxAD$uY3xfNV@(>+MokX7 zb!vbrYclEhsyNYk!5Bk3+`y{%tTUy(N2`p92c6wJKc!WuY^Xj>x3*;n%6yIN;cOO; z(=WlPghhAP_>28W_#nXx1m_5-(D`M8S%L+EG6BVwuM(_@+g$@YX`X>j+^?)EM%ZjO XL#Aa!@VjXEjkvMj2pB!afD!!%SwzEY diff --git a/project/main/manager/fileManager.py b/project/main/manager/fileManager.py index 49d15d5..81e1895 100644 --- a/project/main/manager/fileManager.py +++ b/project/main/manager/fileManager.py @@ -1,7 +1,8 @@ -# charst=utf-8 +# charset=utf-8 import os -import datetime +import datetime, time +import threading from system import shell from base import sysout @@ -10,6 +11,9 @@ TAG = 'filemanager' +# global sizeDict +mSizeDict = {} + def createDir(dir): sysout.log(TAG, 'dir= ' + dir) @@ -56,8 +60,13 @@ def getFileNumber(dir, containChildDir=False): cmd = 'ls -lR ' + dir + ' |grep "^-"| wc -l' popen = shell.SubProcessCmd(cmd) num = popen.getOutBuff() - popen.close() - return num + if not num == None: + popen.close() + return num + else: + # start time counting ??? + return setTimeout(num, popen) + # # get dir number of one dir @@ -71,8 +80,33 @@ def getDirNumber(dir, containChildDir=False): cmd = 'ls -lR ' + dir + ' |grep "^d"| wc -l' popen = shell.SubProcessCmd(cmd) num = popen.getOutBuff() - popen.close() - return num + if not num == None: + popen.close() + return num + else: + # start time counting ??? + return setTimeout(num, popen) + + +def setTimeout(num, popen): + # start time counting ??? + timeout = 10 + t = 0 + while num == None: + time.sleep(1) + t += 1 + if t >= timeout: + sysout.err(TAG, 'Read file timeout for 10s !') + try: + popen.close() + finally: + num = '未知' + break + try: + popen.close() + finally: + return num + # # get only one file name of the dir @@ -107,7 +141,7 @@ def getUserHome(home): shell.execute('ln -s ' + datasetHome + '/* ' + dataHome + '/') all = getAllFiles(home, False) - print(all) + # print(all) return all @@ -180,7 +214,26 @@ def getFilesInfoOfPath(dir): if filePath.endswith('/数据集'): # '/notebook/storage/userId/数据集' filePath = filePath.split('/数据集')[0] + '/system/datasets' - size = getDirSize(filePath) + + # get the dir size + # size = 0 + # if filePath not in mSizeDict.keys(): + # mSizeDict[filePath] = None + # # size = mSizeDict[filePath] + # tmp = 0 + # tout = 10 + # t = threading.Thread(target=getDirSize(filePath)) + # t.setDaemon(True) + # t.start() + # t.join(tout) + # while mSizeDict[filePath] == None: + # time.sleep(1) + # tmp += 1 + # if tmp >= tout: + # size = mSizeDict[filePath] = '未知' + # mSizeDict.pop(filePath) + # break + # info = getAllFiles(filePath) # print(info) # numDirs = 0 @@ -189,8 +242,12 @@ def getFilesInfoOfPath(dir): # numFiles = 0 # for i in info: # numFiles += len(i['files']) - numDirs = getDirNumber(filePath, True) - numFiles = getFileNumber(filePath, True) + + # numDirs = getDirNumber(filePath, True) + # numFiles = getFileNumber(filePath, True) + size = 0 + numDirs = '未知' + numFiles = '未知' d = { "name": c, "size": size, @@ -203,7 +260,7 @@ def getFilesInfoOfPath(dir): } dirs.append(d) else: - print(str(c) + " is unknown") + sysout.info(TAG, str(c) + " is unknown") u = { "name": c, "msg": 'unknown file or directory!' @@ -235,6 +292,7 @@ def getDirSize(dir): size = 0 for root, dirs, files in os.walk(dir): size += sum([os.path.getsize(os.path.join(root, name)) for name in files]) + mSizeDict[dir] = size return size @@ -409,6 +467,7 @@ def deleteFile(path): except Exception as e: return (0, 'Delete Failed, cause: ' + e) + def deleteFiles(paths): result = [] for p in paths: @@ -419,7 +478,6 @@ def deleteFiles(paths): }) return result - # def checkFileNameForm(path): # arr = str(path).split("/") # fileName = arr[len(arr) - 1] diff --git a/project/main/server/__pycache__/httpServer.cpython-36.pyc b/project/main/server/__pycache__/httpServer.cpython-36.pyc index ed1b3a297771b1e301316ae726070394990e7c4e..6e910648b3689600ecf10ec44ecbfff5cfd90604 100644 GIT binary patch literal 7652 zcmc&(%a0sK8SnS>yk{TYjn|Gd>)1}l$$A|JLI?pI?AU9PjE%i{b%G}2sove0o#~!b z*VviWtQBb@4hW$fIpl!2AtZ$O4*-cjfMbgrNGO+x16+LK_fi0dX`uSop_s7aV-}|DbD1T9=eg?`Hk%C24QJBKiuF_K-Ri(1l)f`Py-O(jA90OGE znmx<0Di7g%Vsy*%Q{(Ew$Pq)awunzDxgQ$9GeGSVhd~$beSDtOQ4UjW%d~8QFfHA zfUdA(>^SH#_BcBMdYnDMs-TavRdy2e1bdR50)2v=W@kXF>?!s%=qjtRv!Ey0GwfN= zC)pZ12YQNWx0KrY=LA&dsjuaG zTkUOnYOCFKwy`Z|~^Ip~E+~=z`4fBhp7zClk z-C)oaEm}tM(@?mG6dXt9Dvrt&uu5ebmaQ|L8K4F;v3x6>ulvHi?fbhM>C0O>9-|*o zn(GphoJ%MjwWE!cF+^abj+BNzLahO|V=lE`GsCRd;ck<)z3tH2l*Q0)G}>NUG#a5D zbzZw18eX&K*7VQ_-0oJ|&UgiH`lZ4bKmBTQIBa|CcYVGa^qVbry+6FP z)8xJNWbF}X>oL&ly(UIStXQ95dagg@B}`191Qs$yEvS~dcPyD@g21_C$t}pe{1FVK z4HAkoC=e85WnUElcBFQ6Bx6fklF$fqtdN`o6s^9Ou5vPewH;K6e}Jjl6x9fp>gk}m zhOB1s0$vc7S_95qPc)+L(CqWJCwLis5~on-1Fxp>IU1}A30jmb(vW3jt4Q%yZFN!I zTRLQ`n1mPyE)h?_rCg)Y^Vy(FbDKLKo+DM%Pq zh@d_)#s;%KBaO_|Lu&x$Ut?i576=scaj}j)rj^ZM-~F683>32D@4gj1 z7#ae&h5EKDpb#f%Ai>L@r0f)Bq=k}Wv*6~ube8X8Oi)Gk5Ny?odO|gHAPmAnW?~=%*J&Toui2w_Ymd}7_FW8Q_B2_ol-%w6O@-2m%H1ts9q)e*w zkW^cfvblzlscez~r`1d3&SLckkb@4RZX#Jo8K&;rpjo6GQl4r11<)c=Y50>ESwfhQ zC4|k`8dzivP|tvxpf;!lngz{(=0I)GJZKiQ0Gb0Wg63Iir0iP!7pRv|FR-$#{}%N! z>P1$O_1~jjLA?a4^Q-}jvzz08SC##;s_az+>^?L^9hJxGGT9=u-z&ePc-lLP&^rct zRN0SFBONtE)}TWp3mmsc*-;LydDIJ|;;1w(!E(*j`B^|0ns5zV%xIJp_t3c0lg6bP z1b11OX$QVEXgL&ay1gd+BQ&Nz2=x{+tLOH7J`6L>mVhcg7v}nM=FK*2pxt*pMkR9E zuYS~W`!puh1HmW46=vZb-foMXw|ce8tYA%)a2>1W=P7%RvI~?wPubTf`#NPWQ1%VV zUPKnEvlfwm3(Z}mU<27iTN8^oO%v-F*~m24(vn)xD(Z3dq$+1kWs^J*Im-iDJuHCF z((6p|`DO4~JHTh~G3K*&4?Irz%(P`C;`K88lrTHiNFT#KFnAgjQcrT;i5h(bcyU0OQX?yc*Scp#@B%<(R||d$X9VS8S-k0I zEJud{f>jtDGjaWJR)}S#BP!OVo?Hp7Nan-hz`Nb{*yW~Z2CmqgC{>tYXuHOO$i$Mx z-z&Mjz8F@i1A`G|PeldR468!D>w406g$v0WNK+C!oS9;jH0RO|U&d=3sO_8d2qy;4 zaE)I<+pmxUa(R&+*c26L1<5+dIOJS15CL6IljBqN8F(a>aMGrPeZ$3=9cgSe#f}ZI zLqEU{$Z#y}`aL~n2jVoa;C(tiFjKMsc8*vuPND!SlJwAukmX;BsX2pb1ZV7rVwTm6 zDIl+6ONXLHM&ox#0SRw}8d-@5C!27h5f9+>2YO@&8R7J3mVoR%gdGYZJ-bFY5lYNN zTU4U+fK^9;*hEAR*Qo3rC+)2Iyb6tn4~Ma?dcMF(Z{V>5;uo2Y2k;{M5Y9u-rIugw zw*2c`H!ynw+BNh?cwMBD^z||$nE3aR{Rt^pLN;X?)EtBi@ENcUH0A-=q;;4M;U8VR z$Z00RRMYG5L%bF0ED*4GGd5Cg8pmEblq4H{4a?-OQ+Ak6y@8IuAq5nj%t{BR-gJux zd_=OvN0$u5emWu+@(P;t3VNLra}Jo8!*`ma_)fUQ;l%n+*!m3S;1?;I;cXpl|3FH& z99w<0L^hv#(=8t`8e8ajRArE%kG?Gr55Y~5T5D1ehQk`uB?LwtM_LGL5DE9cGq(IG zEK7RBFh7AKNVtEUdL`p2XrG1R2HJ3BoY;m0iV7^Mbb&M?h2wuG0g&%D{|XY2p{u3K z7#chOAJ8V{b|1?An;24}Jtq>M>{+s(djB8$HMU5Iry2bak&jN=(;|QC%lEvAAuA8v zGxer>CKHnSNruXQ&rePa8Fb!2pk&~*mk2zU+aUgZ1r+R_ehvQ)ISzDqQ!L^^-rhj6#e~N zc0^P-Q)3j)c-7sRNvP`sZ&N+0&62GojD4~u|~Q)h6Bdh3cK2;3jabZq)!*o zqH;wbNQ&nx$(DI>ByB0nmQ!LWZK=qX7sPVfGACO$#A8tlo2M&j83d5)KT_x41I&Bz zaqQdjQb*o9$YTnpbF`CBVo3y`oH!x_7a62WUo5?qwBMYpR%rHiVMH9Ow@bPu85@Nb zPb9a;XOjD4iY$UbzwdLgx_ThGs2z?*xar)iBgVu?`SeC29%I5|x{bVV{*s*F5m zR)?czF!Wk&ANToQFyL-uYal?d%-4Ot@66*k*%)}D-EFk|UZ4D0bb~`ac(N-_28xr$ z$_*uEp3rj)#OAw>g?ag4_7F)Ieu&&Da(t2Gp8^5}XB`6}^P)L99zLX`BT$sF58@x9 zvq|IcVZN3|v*Jfaq>3c^M)&%>LK^VD@~cL?^ON87o(DS%~YF=)w@zfOc+f!v@p} z^*!st`SaE5uiY@~UxyA?|Xq;rGA*%_rW6m zLu9zYpg>n%uJa>QqH}j>(D#Z^r>pg+sZmCo@1o?C5Lh?dJ9Gu#h4gYq4yPrDX4h}x z=t%MqnrzVP<3o(=(ft}esbpO5PMiClQkc1+5p;LKZ571#Pk%@uxPq2L1sC zM|>s5g~n}Ifyj^{c9LJ+b4DHs;0x`-^aUyeS*e1}#Ykk}N! vYPtY`AkZI#!NLauThE#EbM}J0Y%kg8?K1K?`-ok%3w8;qV9(nZ?CQS&@?Ha( delta 3022 zcma(TNo*TcbpCi|Jelzp$FZ}xN!m1vo3y1V>6WgvP1TLIYDwBbC^-3()Q&g)Oj|Wd z@geL8vHTlCNC;8E8G1oN>H&#ED=5bf7bKPorxp$fp^EpuQ^#=))YgCR&HHcO{{Ekz z$8L<4w+4g8^-q6)cE^Au{Up`=d_eDq<=h8AB}#Hqo>+t+E$3uQ2JFlE@`|Mho|04Z znxzR^&H3|w%P(jR_`0P7?YEk2f7Y-}fB`FL2W@kz%klNi~DsUb-G|lJ?Plz-#D0k`4mVMN@PG;BGoZhXMCc`K*-M$aj<1_;bo{T_&W!Q%c#R z)YD4Yr_*`c0(|aj$^J}YG5(%B8S>0bA>*EA#jHK!cJs2@OXB=v^&=DCWdQ_1eo9Nk z!a$~E)`T1bX_ZCzf;PCb8972aic|}N1Ok*|u_|Z<$}C zmTQcmJVEe?K_81CUjbkRshtCdm;FxZtj-JJ0J`ESGev59zPvM6@x9I=p~!IUWlcwU ztDYiVpYPL;lVm5puR3m;xs?Db=|CqE{CZQCM0m3ipRO~#l7JqRT#H~Gf?fpc5%eMG zM*!U`u|WhW03NaH62^vsJ;h%)&X8eFTH5&^My?;T^%vRvXomc31F9NA5aF+w17}uz z+Xzfgxme6YGMMeu+3}1y+dX-8Jd?AXw?WduT1rGAG6|3fNy<@T$TDFg{DJu$N$_t2 zYqUzy_`SfWS!;Gnuy1U|tVo%t#waMX{2Aw&=}dt#bf?Y-wjFpM@^^znWIO*V_!cqw z@zCazHM;Q~EWanJ=GuqSZrZWklu}dv9VpaD`%c*Sf`1cQLw17pcK#srNmTThx+-fo z!rgTaa4aH*5@eU-Q4i-DFH?8;xY2uk%y`5l>@w0p{+Zi@Qmo{m4%R5FeX#MX#Bayi$UYI5 z8NF0~WD>R7jG?MF1R`E@#jNu%=>51W;x&+}%j5yz;aTkU5r-aeFJ4?}zkR8&m;YlsKY48Ij?CkOexmN!o|QftgGJH0x2hyGvlVGs!j&4>9f zv6su@74+ya%BAOvmn+56D#fb4{6FE0gZ?%+FstdS_DnBHk_Zkhip=1dia!~a3JcTV zY~z9Wm-9E>;G(od&Pt1f2IggU#|>4VgYf*L8?HWw=H+)JcHfOuIbq-srnt>jPDF6x zZnVm27Mwx1g`Y{ZO>YI$3;y(s3%9@?7KbO+fdJ1s)&;;a@r+oWZuTUyaeCPZ0-QXC zlftlB8Mcq7=I3dKSz3O67W$oy^IM5+A%oaW0xdTTkM`Q!b;tcw4 zV+=FLu+}26P+Kuv7}ToPhI_zkDls^qFpv#cqH%t=Z40i#{BWOfWx%wT=+ zhtt#pOVsI4AfPyensa1H#_yABvMVi0RB4y8#FZDNtdFWxTauT^5~2Pj8TJ*bFTr=u z)rhNrtk@yaq*NGDBv+j`%4~s1*L*};h=XVoin_|85AmdQ0Y8^u_eI}%sUV-1TrKNI z6`)oZSXox9put+T?9O9qZLg?pS_BnY1AHdeRB*&_C4R8u{juW^GCKhvWr}Kf3XCN6 ze3^pF+JWtk?cFN`h0`i_EO;m zRE!msWV}w91ypLS)q2I5%P@O(latApa`qIsc-dwQTY#PBx03_qqdz3GgWch)0vFNyL5@jzS5PiPV(l|$zi4s$TcWb4u8_Z8#ON&Ni7}m7#w5F{^Y~o8! z-<>bgOF4Tl9wyE<00E*ybm9-m(}+fNpJD1PdP0xuPv~KQ5xrFp>H$3jE1*a9VZHMo DX!JAY diff --git a/project/main/server/httpServer.py b/project/main/server/httpServer.py index 97da85b..e9aab9d 100644 --- a/project/main/server/httpServer.py +++ b/project/main/server/httpServer.py @@ -4,6 +4,8 @@ # from http.server import BaseHTTPRequestHandler, HTTPServer from wsgiref.simple_server import make_server import json +import threading +import asyncio from base import sysout from manager import projectManager @@ -19,8 +21,19 @@ # global response data resp_err_params = {'status': '0', 'result': 'request params form error!'} +global mLoop +class NotebookHttpServer(threading.Thread): + def __init__(self, threadId, name): + threading.Thread.__init__(self) + self.threadID = threadId + self.name = name + + def run(self): + t = threading.current_thread() + sysout.info(TAG, 'NotebookHttpServer is stating at thread %s - %s' %(t.threadID, t.name)) + run() ## # module -- project @@ -193,6 +206,8 @@ def deleteDataset(request_body): # # get all files & dirs info of the current path # +# @asyncio.coroutine +# async def getFilesInfoOfPath(request_body): def getFilesInfoOfPath(request_body): path = None try: @@ -323,6 +338,12 @@ def praseData(request_body): return getMyFiles(request_body) elif action == 'getFilesInfoOfPath': return getFilesInfoOfPath(request_body) + # loop = asyncio.get_event_loop() + # task = getFilesInfoOfPath(request_body) + # futrue = asyncio.ensure_future(task) + # loop.run_until_complete(futrue) + # return futrue.result() + elif action == 'rename': return rename(request_body) elif action == 'deleteFile': @@ -343,31 +364,35 @@ def application(environ, start_response): # 定义请求的类型和当前请求成功的code # print(environ) # print(start_response) + sysout.info(TAG, threading.current_thread()) start_response('200 OK', [('Content-Type', 'application/json')]) # environ是当前请求的所有数据，包括Header和URL，body request_body = None # request_body = environ["wsgi.input"].read(int(environ.get("CONTENT_LENGTH", 0))).decode('utf-8') request_body = str(environ["wsgi.input"].read(int(environ.get("CONTENT_LENGTH", 0))), 'utf-8') - print('http request ---> ') + sysout.info(TAG, 'http request ---> ') try: - print(str(request_body)) + sysout.info(TAG, str(request_body)) except Exception as e: - print('Exception:') - print(e) + sysout.error(TAG, 'Exception:' + str(e)) + + # global mLoop + # mLoop = asyncio.get_event_loop() + request_body = json.loads(request_body) response = praseData(request_body) try: - print(str(response)) + sysout.info(TAG, str(response)) except Exception as e: - print('Exception:') - print(e) + sysout.error(TAG, 'Exception:' + str(e)) + result = json.dumps(response).encode('utf-8') - print("http response --> " + str(result)) + sysout.info(TAG, "http response --> " + str(result)) return [result] def run(): # httpd = HTTPServer(mServer, AmiHTTPServer) httpd = make_server(mHost, mPort, application) - print('http server is running on ' + str(mServer)) - httpd.serve_forever() \ No newline at end of file + sysout.info(TAG, 'http server is running on ' + str(mServer)) + httpd.serve_forever() diff --git a/project/main/system/shell.py b/project/main/system/shell.py index b27fff1..1001436 100644 --- a/project/main/system/shell.py +++ b/project/main/system/shell.py @@ -1,14 +1,14 @@ - import subprocess, os, time -from base import sysout - +import traceback +import tempfile +from base import sysout TAG = "SHELL" def execute(cmd): - sysout.log(TAG, "executed: ["+ cmd +"]") + sysout.log(TAG, "executed: [" + cmd + "]") return os.system(cmd) @@ -16,17 +16,43 @@ def execute(cmd): class SubProcessCmd(object): def __init__(self, cmd): - self.popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) - pid = self.popen.pid - sysout.info(TAG, 'Popen.pid:' + str(pid)) + try: + self.tmpOut = tempfile.SpooledTemporaryFile(10 * 1000) + self.fileno = self.tmpOut.fileno() + + # self.popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + self.popen = subprocess.Popen(cmd, stdout=self.fileno, stderr=self.fileno, shell=True) + + self.popen.communicate(timeout=10) + # self.popen.wait() + self.tmpOut.seek(0) + pid = self.popen.pid + sysout.info(TAG, 'Popen.pid:' + str(pid)) + + except Exception as e: + sysout.err(TAG, str(e)) def getOutBuff(self): - while True: - line = self.popen.stdout.readline().strip() - # 判断内容是否为空 - if line: - return line.decode('utf-8') + try: + # while True: + #old 1 + # line = self.popen.stdout.readline().strip() + + #old2 + # line = self.tmpOut.readline().strip() + # # 判断内容是否为空 + # if line: + # return line.decode('utf-8') + + #new 1 + return self.popen.returncode + except Exception as e: + sysout.err(TAG, str(e)) + return None def close(self): + self.tmpOut.close() self.popen.kill() + + diff --git a/project/test/asyncioTest.py b/project/test/asyncioTest.py new file mode 100644 index 0000000..4e75c29 --- /dev/null +++ b/project/test/asyncioTest.py @@ -0,0 +1,54 @@ +import asyncio +import time + + +async def run(a): + print('this is task for num %s' % a) + # await caculate(a) + res = caculate(a) + print('task for num %s is done...' % a) + print(res) + return res + + +def caculate(a): + i = 0 + while i < a: + i += 1 + print(i) + time.sleep(0.05) + print('i=' + str(i)) + return str(i) + + +def callback(future): + print('callback res= ' + str(future.result())) + return future.result() + + +if __name__ == '__main__': + + # print(asyncio.iscoroutinefunction(run)) + task = run(5) #sigle task + # futures = [asyncio.ensure_future(run(5)), asyncio.ensure_future(run(2)), asyncio.ensure_future(run(10))] + # tasks = [run(5), run(2), run(10)] + + loop = asyncio.get_event_loop() + + try: + ##single task + future = asyncio.ensure_future(task) + loop.run_until_complete(future) + print('res=') + print(future.result()) + # future.add_done_callback(callback) + + ##multi tasks + # loop.run_until_complete(asyncio.wait(futures)) + # # loop.run_until_complete(asyncio.gather(tasks)) + # for t in futures: + # print('res=') + # print(t.result()) + # print('Done after {}s'.format(time.time())) + finally: + loop.close() From 5271d1462da8ce91a1985cf6ef27eae0d54ae21a Mon Sep 17 00:00:00 2001 From: JerryYin Date: Thu, 22 Nov 2018 11:29:53 +0800 Subject: [PATCH 14/42] releae version for FilesSystem --- .idea/workspace.xml | 213 +++++++++++++++---------- project/main/config/config.py | 46 +++--- project/main/manager/fileManager.py | 14 ++ project/main/manager/projectManager.py | 2 +- 4 files changed, 163 insertions(+), 112 deletions(-) diff --git a/.idea/workspace.xml b/.idea/workspace.xml index 3b57cce..50f653d 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -2,13 +2,9 @@ - - - - + - - + - + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -112,8 +141,8 @@ - - + + @@ -124,8 +153,8 @@ - - + + @@ -140,25 +169,25 @@ - + - - + + - + + - - + + - - + + - - + @@ -167,8 +196,8 @@ - - + + @@ -193,16 +222,6 @@ - makeDir - ns_doname - deleteFile - ln - - ln -s - system/datasets - getUserHome - dir_home - initProject - dir dele createPreProject createDir @@ -212,7 +231,6 @@ subpro subprocess getFileNumber - getDirNumber size syso ill @@ -223,6 +241,17 @@ PRINT print getDirSize + utf + sysout. + .log( + cmd is none + start vm failed! cause: + rm -rf + resp_err_version_invlid + deleteFiles + getDirN + incorrect version number! + getDirNumber @@ -251,16 +280,16 @@ @@ -291,6 +320,11 @@ + + + + + @@ -787,20 +821,6 @@ - - - - - - - - - - - - - - @@ -808,16 +828,6 @@ - - - - - - - - - - @@ -886,27 +896,41 @@ - + - - + + - - + + + + + + + + + + + + + + + + - - + + @@ -919,40 +943,53 @@ - + - - + + + + + - + - - + + + + + - + - - + + - + - - + + - + + + + + + + + \ No newline at end of file diff --git a/project/main/config/config.py b/project/main/config/config.py index 42e37b4..6ca9892 100644 --- a/project/main/config/config.py +++ b/project/main/config/config.py @@ -21,42 +21,42 @@ vms_port = 18883 vms_host_dev = '121.40.62.80' #dev vms_host_release = '60.12.136.59' #release -vms_host = vms_host_dev +vms_host = vms_host_release # ----------- RELEASE ------------ -# ns_host = '172.16.59.99' #_release server cloudyotech.com -# ns_port_http = 8100 -# ns_port = 8888 -# ns_doname = 'https://g.cloudyotech.com/notebook' -# # ns_doname = 'http://' + ns_host + ":" + str(ns_port) -# -# # system home dir -# dir_home_release = '/notebook/storage' -# dir_home = dir_home_release -# -# # dir_home_user = '/users' -# dir_home_user = '' -# # user's project dir: homeDir/users/userId/projectId/version/xxx-nb -# -# file_system_readme = dir_home +'/base/readme' - - -# ----------- DEV ------------ -ns_host = '172.16.3.254' +ns_host = '172.16.59.99' #_release server cloudyotech.com ns_port_http = 8100 ns_port = 8888 -ns_doname = 'https://dev.dongxicc.cn/notebook' -## ns_doname = 'http://' + ns_host + ":" + str(ns_port) +ns_doname = 'https://g.cloudyotech.com/notebook' +# ns_doname = 'http://' + ns_host + ":" + str(ns_port) # system home dir -dir_home = '/notebook/storage' +dir_home_release = '/notebook/storage' +dir_home = dir_home_release + # dir_home_user = '/users' dir_home_user = '' # user's project dir: homeDir/users/userId/projectId/version/xxx-nb + file_system_readme = dir_home +'/base/readme' +# ----------- DEV ------------ +# ns_host = '172.16.3.254' +# ns_port_http = 8100 +# ns_port = 8888 +# ns_doname = 'https://dev.dongxicc.cn/notebook' +# ## ns_doname = 'http://' + ns_host + ":" + str(ns_port) +# +# # system home dir +# dir_home = '/notebook/storage' +# # dir_home_user = '/users' +# dir_home_user = '' +# # user's project dir: homeDir/users/userId/projectId/version/xxx-nb +# file_system_readme = dir_home +'/base/readme' + + # ----------- DEV2 for 18.18.18.174 ------------ # ns_host = '18.18.18.174' # ns_port_http = 8100 diff --git a/project/main/manager/fileManager.py b/project/main/manager/fileManager.py index 81e1895..9dc146d 100644 --- a/project/main/manager/fileManager.py +++ b/project/main/manager/fileManager.py @@ -108,6 +108,20 @@ def setTimeout(num, popen): return num +# +# get the directory number int the 'dir' +# linux : +# get dir number of the dir +# #ls -l |grep "^d"|wc -l +# +# get file num of the dir +# #ls -l |grep "^-"|wc -l +# +def getDirNumber2(dir): + cmd = 'ls -l ' + str(dir) + ' |grep "^d"|wc -l' + n = os.popen(cmd).read() + return n + # # get only one file name of the dir # --fileForm: .ipynb / .html / .py diff --git a/project/main/manager/projectManager.py b/project/main/manager/projectManager.py index f904c32..2e29abb 100644 --- a/project/main/manager/projectManager.py +++ b/project/main/manager/projectManager.py @@ -21,7 +21,7 @@ def checkVersion(userId, projectId, v): if v == None: return (False, resp_err_version_invlid) - maxVersion = int(fileManager.getDirNumber(config.dir_home + config.dir_home_user + '/' + str(userId) + '/system/' + str(projectId) + '/')) + maxVersion = int(fileManager.getDirNumber2(config.dir_home + config.dir_home_user + '/' + str(userId) + '/system/' + str(projectId) + '/')) if (v <= 0 or v > maxVersion): return (False, resp_err_version_invlid) return (True, maxVersion) From 0db025a25222e82a4ace582ac08a3120f1cdea23 Mon Sep 17 00:00:00 2001 From: JerryYin Date: Thu, 22 Nov 2018 15:19:03 +0800 Subject: [PATCH 15/42] improved config, one setting to change mode --- .idea/workspace.xml | 367 ++++++++---------- logs/NotebookServer.log | 0 .../base/__pycache__/sysout.cpython-36.pyc | Bin 903 -> 1430 bytes project/main/base/sysout.py | 19 +- .../config/__pycache__/config.cpython-36.pyc | Bin 1683 -> 2922 bytes project/main/config/config.py | 171 +++++--- .../__pycache__/fileManager.cpython-36.pyc | Bin 7656 -> 8273 bytes .../__pycache__/jupyter.cpython-36.pyc | Bin 1211 -> 1211 bytes .../__pycache__/projectManager.cpython-36.pyc | Bin 5948 -> 5949 bytes .../__pycache__/vmManager.cpython-36.pyc | Bin 2006 -> 2006 bytes .../__pycache__/httpServer.cpython-36.pyc | Bin 7652 -> 7743 bytes project/main/server/httpServer.py | 9 +- project/main/{main.py => startup.py} | 2 +- .../system/__pycache__/shell.cpython-36.pyc | Bin 1210 -> 1522 bytes project/test/__init__.py | 2 + 15 files changed, 285 insertions(+), 285 deletions(-) create mode 100644 logs/NotebookServer.log rename project/main/{main.py => startup.py} (95%) diff --git a/.idea/workspace.xml b/.idea/workspace.xml index 50f653d..ccb2543 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -2,9 +2,13 @@ + + + - - + + + + - + - - - - - - - - - - + - - + + - + + + + - - - - - - - - - - - - - - - - - - + + - + - - - - - + + - + - - + + - - - - - - + + + + + + @@ -171,11 +151,11 @@ - + - + @@ -184,10 +164,10 @@ - - + + - + @@ -196,17 +176,20 @@ - - + + + + + - + - - + + @@ -222,9 +205,6 @@ - dele - createPreProject - createDir createProject getAllFiles ls -l|grep @@ -248,10 +228,13 @@ start vm failed! cause: rm -rf resp_err_version_invlid - deleteFiles getDirN incorrect version number! + ns_doname + deleteFiles getDirNumber + mPort + ns_port_http @@ -283,13 +266,15 @@ @@ -320,16 +305,6 @@ - - - - - - - - - - @@ -341,20 +316,6 @@ - - - - - - - - - - - - - - @@ -362,12 +323,6 @@ - - - - - - @@ -354,7 +321,7 @@ - + - + - + @@ -508,16 +475,16 @@ - + - - + + - + @@ -565,13 +532,6 @@ - - - - - - - @@ -804,9 +764,6 @@ - - - @@ -814,9 +771,6 @@ - - - @@ -831,15 +785,9 @@ - - - - - - @@ -847,93 +795,102 @@ - + - - - - - - + + - + + + + - - + + + + + - + + + + - - + + - + - - + + + + + + + + + + + - + - - + + + + + - - - - + - - + + - + - + - - - - - - - - + + + + + + + + + - - + + - + - - + + - - - - - - - + diff --git a/project/main/base/codeListener.py b/project/main/base/codeListener.py index e69de29..3ac14a6 100644 --- a/project/main/base/codeListener.py +++ b/project/main/base/codeListener.py @@ -0,0 +1,13 @@ +import inspect + +def getCodeLocation(): + ins = inspect.stack()[0] + file = ins[1] + lineno = ins[2] + function = ins[3] + code = ins[4] + location = ('NotebookServer' + str(file).split('/NotebookServer')[1], lineno, function + '()', str(code[0]).strip()) + print(location) + return location + +getCodeLocation() \ No newline at end of file diff --git a/project/main/base/sysout.py b/project/main/base/sysout.py index a053be9..b4db845 100644 --- a/project/main/base/sysout.py +++ b/project/main/base/sysout.py @@ -24,8 +24,7 @@ def log(tag, msg): print(log) logging.log(log) except Exception as e: - print('Exception:') - print(e) + print('Exception:'+str(e)) logging.error(str(e)) @@ -36,8 +35,7 @@ def err(tag, err): print(log) logging.debug(log) except Exception as e: - print('Exception:') - print(e) + print('Exception:'+str(e)) logging.error(str(e)) @@ -48,6 +46,5 @@ def info(tag, msg): print(log) logging.info(log) except Exception as e: - print('Exception:') - print(e) + print('Exception:'+str(e)) logging.error(str(e)) From fddf119562e23ec7404e9712281c401fd66f9422 Mon Sep 17 00:00:00 2001 From: JerryYin Date: Mon, 26 Nov 2018 16:28:20 +0800 Subject: [PATCH 18/42] done initClass's step 1, [copyClassProject] --- .idea/workspace.xml | 357 ++++++++++-------- bin/autodeeplearning.sh | 4 +- .../base/__pycache__/sysout.cpython-36.pyc | Bin 1430 -> 1437 bytes .../config/__pycache__/config.cpython-36.pyc | Bin 2922 -> 3176 bytes project/main/config/config.py | 25 +- .../__pycache__/fileManager.cpython-36.pyc | Bin 8273 -> 8442 bytes .../__pycache__/projectManager.cpython-36.pyc | Bin 5949 -> 7311 bytes project/main/manager/fileManager.py | 6 +- project/main/manager/projectManager.py | 119 +++++- .../__pycache__/httpServer.cpython-36.pyc | Bin 7743 -> 8345 bytes project/main/server/httpServer.py | 54 ++- .../system/__pycache__/shell.cpython-36.pyc | Bin 1522 -> 1522 bytes 12 files changed, 384 insertions(+), 181 deletions(-) diff --git a/.idea/workspace.xml b/.idea/workspace.xml index 6c712d2..1e84e27 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -2,8 +2,13 @@ - - + + + + + + + - + - - + + - + - - + + - + - - + + - - + + - + + + + + + + + - + - - + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -158,12 +209,6 @@ - createProject - getAllFiles - ls -l|grep - subpro - subprocess - getFileNumber size syso ill @@ -185,9 +230,15 @@ incorrect version number! ns_doname deleteFiles - getDirNumber mPort ns_port_http + os.list + os.l + getAllFiles + copyClassProject + getOneNbFileName + getDirNumber2 + getDirNumber @@ -207,7 +258,6 @@ @@ -251,7 +303,6 @@ - @@ -259,6 +310,16 @@ + + + + + + + + + + @@ -282,18 +343,20 @@ - + - + + - + @@ -407,7 +452,7 @@ @@ -538,16 +583,16 @@ - + - + - + @@ -595,73 +640,6 @@ \ No newline at end of file diff --git a/project/main/base/__pycache__/sysout.cpython-36.pyc b/project/main/base/__pycache__/sysout.cpython-36.pyc index c66d3fa626a1b9da69f608f3edbb779fc2b8f947..75c6edfb3c6b3357aada6eb7fb59729f497835bf 100644 GIT binary patch delta 339 zcmYk1zfJ-{5XNVA?_jy*;P4LzXkl|kV!*<}NM&iGF`;s;)L9cx$dNl@B8|n_SeRgF zwX&w-C47dn)6Q40alyu!eDj+x-!$_ozvca|>pCCrKWC3Nz!&^0VfF)~WNZcjVGqYZh7rbqH-@sY&ZV+y|A@5ax{pS!C-R<$TaQEKuG|Cxw*x`#{16DvIB`9HCx9q;_7V5)(|%)Q&xZb#Vn!Bvfi!RT5i(t<_sn)C-6Ua{{1A%*+})CNmo delta 262 zcmYk0u};H45JY$HeD=i`2P3Qm3n5xULy;a)QiUSjAyJ^Y#vmEl5mG9OK#2r$O~ogq z;ak$ApyLNKqMVZzeyyYirTpt2z*2SkL>yPR$c{F zX;4X31htg5B^t-y$U9^h2>aMVF3i9) z8gIY6-{w2V(|R7R>iT*Xn!C`fru@uYtTt+o`JmgV@~w!NkA6}ylwZe##tGJ{Zr8e9 awB4CG>>kbeFxR`vvXYI3sfx69)_nuTKQ#{k diff --git a/project/main/base/sysout.py b/project/main/base/sysout.py index b4db845..06eede4 100644 --- a/project/main/base/sysout.py +++ b/project/main/base/sysout.py @@ -1,15 +1,16 @@ # coding:utf-8 import logging -import os, sys +import os, sys, time + # from config import config # path = os.getcwd().split("/project")[0]+'/logs' -path = sys.argv[0].split("/project")[0]+'/logs' +path = sys.argv[0].split("/project")[0] + '/logs' logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s', datefmt='%Y/%m/%d %H:%M:%S', - filename= path + '/NotebookServer.log', + filename=path + '/NotebookServer-' + time.strftime("%Y.%m.%d-%H:%M:%S", time.localtime()) + '.log', # filename= config.logpath, filemode='w') logger = logging.getLogger("NBLogger") @@ -24,7 +25,7 @@ def log(tag, msg): print(log) logging.log(log) except Exception as e: - print('Exception:'+str(e)) + print('Exception:' + str(e)) logging.error(str(e)) @@ -35,7 +36,7 @@ def err(tag, err): print(log) logging.debug(log) except Exception as e: - print('Exception:'+str(e)) + print('Exception:' + str(e)) logging.error(str(e)) @@ -46,5 +47,5 @@ def info(tag, msg): print(log) logging.info(log) except Exception as e: - print('Exception:'+str(e)) + print('Exception:' + str(e)) logging.error(str(e)) diff --git a/project/main/config/ws_conf.json b/project/main/config/ws_conf.json new file mode 100644 index 0000000..c2b641d --- /dev/null +++ b/project/main/config/ws_conf.json @@ -0,0 +1,4 @@ +{ + "ws_host": "127.0.0.1", + "ws_port": "8101" +} diff --git a/project/main/manager/__pycache__/fileManager.cpython-36.pyc b/project/main/manager/__pycache__/fileManager.cpython-36.pyc index a2592651a6da05e23b29ad4f0a14f56c0823b306..b95b874afb5cba82e02e534952ae81cf11d72538 100644 GIT binary patch delta 37 tcmez6_|=iqn3tC;^wpney^WlUBqxhWWKLcoEIwIVYBD3s<|9&OYyb&!4A%ev delta 105 zcmezB_{)*gn3tF9$;#i+IvY6`NxGFVq%dSN74f8S1~a5EmoSzvG&5$i7s;e>1v69% zl`v&7r?A#Ahyz6#8B(}|8CV!fShBf_q8J%!fg)7`DU2x`C5)3lN}gq7E#hHdVA#B0 H%8U&F@_-sG diff --git a/project/main/manager/__pycache__/projectManager.cpython-36.pyc b/project/main/manager/__pycache__/projectManager.cpython-36.pyc index cb67e74f09e1a0681e777d8df64554162680c3e6..dfe33a008c31ea6ea952c96619e2a7004b6bdf07 100644 GIT binary patch delta 97 zcmeCT?6u@H=H=yTUiUjXXd|aIGdB+d5HNB8q0weTW`1TyrOl3Po0%CsH-F_k!Niz5 zc^}U=#yOj}^2)L@F5P@tD2bVod9#_A6f2|Y=0IsfMqXYHbq-Z7Jq}K;I1VEKvF#Tx delta 99 zcmeCR?6>4J=H=zOyXJRv^hQo;W?nu9AYkMGLN-R7&HBvz%#6yL?b$XnGkR_Q%z1)| zF>mr7o^OnEH*e;ZWo2Bp`Jzw~Gb77p6EP`PMzhWS(uRz@d>rZ=s$6;;oLq4nMgZTD B7mWY_ diff --git a/project/main/manager/fileManager.py b/project/main/manager/fileManager.py index fecf006..6e17d71 100644 --- a/project/main/manager/fileManager.py +++ b/project/main/manager/fileManager.py @@ -409,9 +409,9 @@ def makeDir(dir): def createFile(file): - if dir == None: + if file == None: return (0, 'File can not be null !') - if "/" not in str(dir): + if "/" not in str(file): return (0, 'File form not support, should be absolute path !') else: try: diff --git a/project/main/server/__pycache__/httpServer.cpython-36.pyc b/project/main/server/__pycache__/httpServer.cpython-36.pyc index 361d76819e866d6c5b90798e4625521831655ca0..3a10feb550064e1d4da4e6a66eecaf522e8588a4 100644 GIT binary patch delta 2617 zcmai0OKcNI7~WaGHg;?r+p&`v$8JJM3<1&rt!RR(k`Nl80qTT@g8&mR#7_Ls>;eKI zL4^RR?V)r|y-<2-)k6=ILocWtd+I|y^suK^Ahl9ewN)R7Qc(YIY!cfz#jE{1yZ_Ah zKjxd6eead;cDOftz3xwc`FHlC=Wn~;P=l6c&T~DJoy>B-q9`9LJSc9eM_Gq>s6Ge< z=@pj{LaT!yCjPLcy3wi_8oGGM5D{7XqqQ+=S@v>%GHW@@`Ke4nPt*VO`LbR!FeI)! zwy?G0uH%CjFo1Uh;uz{Wt8*h)D)JV*Vu0rel4|9J8*K;g5yCl;?1fd1UxQX3pdWx< zb?zQGx3P7i+jV`Q-rWigY~&#O*uqV8)X~s9M<-* z)$7sMVVD_iR^nXBllO~zE&0J71ZqdG7JjC9C}++eEF}HB!EYrok#-Mt@@?qM>WT@D zQE}5f!$!rQ?yGVN{$ZP*uA7U$guWwy9`U}Xzi-*mcJSKgO9e`c>qcGme)ghc9H56{|R!=SAZg1a<%Y-sjS+Si&op$AnnUlFQMBe50e%j; zu~m=JZKOqC_{gQKl2VfPsS2ND25HjPFvm|Of;W9a67a&H8P zRbh5ql%z&iEAunX(&AcV{O|;PR@Dy|X|1}08)*lQfx4}P>{c6XFS;qc?B}JtK4jP8LuVXvcv1_bRZ|N;&S|Tq%xTV$%0G1OXi^4bip2)DA$LmK~o2mMvE_c@i(N#4U%;d5K&0yiqDzj>S@jX7reN zH@0zWFZw{jmPXT1E)@;^sgoC0Po`X6%*|ym2e-vvu?^8Yy`H06{&4)c=tfS)l-S_-Zaa{DMXTIB4;FurE}s|_e48lX{^hLc-TEM z*l;4?;RH&&X-i{Amk^+#T@WOy^q#6Z6d=(i= zsV(cFiRhY1fu}(*3pffm29U{AztSko8i|C&&P4wPo7)ixXhAKag|#7V75xXP<> R_RqNA#!se*OD5AI|R``dGiNhErO3i5uz6c!mcRMR{N0A#q2&z@p-z`k*O9 zQ(QtAqXaTUTjROZc_{!5ZT<1V?!;9ryEM^SLkvCti$g-tFv}=dhz}lR4lErpo zt;;N@dAXuW^`Rz<)g%)fBjS!e z%SOas`b}AazuDm9dGqiCSbGrACEoV-^;8`lf(YkQF;8J}!>q~PFWx=uh=};UVZGvK zUx&MrL!tZkdF_M~{+|6+LK&*e*eFq_JXv%0Vz!XsaAar4RQ|8 zM10T~8IVy|weA7;VL+YWNt*jZ{N5O4lTvVZ=<==v&DGGm?Wq7spST-ptsJE1@hnx> zA0HUb5UeZWQ7FdFkoMP3*>u7#li|symyOXZr<6t<1cEw1#E4nu~SMsr~_*tUp zX5z;{#2r*s#%G1M2M00k-OqIWp`QEYiW|#D}fP7|KjB z$b8D}wz`Y^k!JsKx+G1NRw<017opbCx#zHDqam>+rEOfQX3B+|x#o0}-D;#0ZyP2BuiD4Jy#kJ6?q zp7dgwZuw_<4=9p=5x@XJ%8QF)gMlBzG|n|Y0>HWCI7b`}FWb^`<(D&Ao^t1xFHsSx z3O}?RyVeJD7r+5%2DAX=^|ysvJ_hT5dgAC-j#lF+Jx8tntqsDml!^t@c%1W*wqehiWYKwA1Wy)1YPQRfrC8JC$Nyp@sfte*EK|ZP95{2R` ziL5E+JI9(qULWW|@k!^%Zd)^B4SryJhL%AZ6xGe#Ok+MZ&Ui$0BzEdh$YD!~@x&9? zWIbh#{V;d|)ykrmDqSTzr&``EdT6W4^79}_0p|N=VMuG%{F+Y-(8ouXpyt(d8Vzfm{{R^q!q)%* diff --git a/project/main/server/__pycache__/websocketServer.cpython-36.pyc b/project/main/server/__pycache__/websocketServer.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6a956d2e9e6b6cc63c69a0e730eed0fd5c2d5978 GIT binary patch literal 2370 zcmZ`)&2Jk;6rb5IukE$-)h4Z4V2fIip^c-oLZB8XN)hy8L6fRzL$%uOOcHPGU3X^O zVB<)Y+&gdtA@1dm-~xX_PjleXQxE(D6$jp%O+p&Mtme&|nKvK%-tYb9YNJv6^{>A- zfAa}>PL7oZ=r^ItKS40UXpcl?OCxIfR%Dshj%?F9kz-mna-p?)UcVAmD6X-4e&3IL z(4DBtYs_Wd)cS5Vs&hLFczsHf#yvRUOLB*NMR+A^@+zxLEqb8y8P%9l?5hUz&yr{s z+A6ejyq?WZ$)O#q@fkkL=eDdVT_R}z8DVwiJb`n$n=MT3=oAZB;|YmQvnHE?c9GF_ z(wcn%|42G5sy#W9#ZaM^7Wr0pThHwA2kS+;%hfs;kGL2wloRbe8KbtG$s*Sj#=Ch} zg?lnA=a-k4!!W9V)lMOmQ3i!j<2PP43t5r#%dpItg&&3~UgfY@pxlHi&x7Cu9S=^Y z%z~@i%*I~>r@QbAo^Ze*Hy*n)Oi zp027L!4uZax3#xnRPDzx-Z_r7Urye>rJX$KbM5qsV$gE5BYAJ@XqJf@94P7_o)Xc7 z=HuYWEYchPB}N zHbg=mi^TF^BmlMKb*K^*LIdi-H$HPBX3K|vkWw^*g1&K~8viVVg8rG3eX6X?Rt}>R znz>NDEo;#rNmV8!^U-_8S5{|y`NVa?t_-D06u2%-R9Lzyyd=XbAX-)7!6~|#4u#;k zipyz%_s~u^-zv1d@yTbR3Vq`~ZHr-UeAu!?fL2r)UzTD9?G`{>geq}sLVbFMju%e; zI&9NvxfnzY83eHw$Nhp0d#DF-{BW4`${FKKv+1#eFnq$ChLA!SN>qHy_fot(mI_oH z{2i(cz{TiH1vteN1h?5!avR+CIoY?swH7!Q5)@sj9OWJ9mCQ#SZT=md0LAQD0TItY zv(G{z?^@y+?Bd}1Z+=l#=1%D?a{udndtx(h*V=bf4YhNGd8YvGtg=KV4)cGu_Fd&n zz}noYt}0o8y(um42BVJn18&7(sp*J2%2__qCU#s$1ldpB97Btwbd+IC$~}1M!kx zYFWAh*^qK6wS#Q2{nL+Fo_;*8-r*02T&i$$eEL3~kQd6-q}MCUVSfMXjOFeC`Yt2NYsw@MR{YXX;-{4LfgFw@ z4?(B>A}I3M*d8dIYP< z`VCMJs3j+sd9PgMO>JNS03$#Ph*s5b3||yg&jN%XGD7*fAXp{_$e=Y5HYVUsXBfiX z=QdYYuU}oger=`y{PFjzD=Xpp-Ll|+v6v|H$=423) z?jlyrYC^ouLp3w4C2b_YcO diff --git a/project/test/__init__.py b/project/test/__init__.py index 5d47134..9347927 100644 --- a/project/test/__init__.py +++ b/project/test/__init__.py @@ -1,2 +1,7 @@ import os -print(os.getcwd()) \ No newline at end of file +import datetime +import time +# print(os.getcwd()) + +print(time.asctime( time.localtime(time.time()) )) +print(time.strftime("%Y.%m.%d-%H:%M:%S", time.localtime())) \ No newline at end of file diff --git a/project/test/wsClient.py b/project/test/wsClient.py new file mode 100644 index 0000000..e367fb2 --- /dev/null +++ b/project/test/wsClient.py @@ -0,0 +1,38 @@ +import asyncio +import websockets +import json + +params = { + 'action': 'copyData', + 'userId': 1, + 'classId': 1, + 'projectId': 10, +} + + +async def hello(uri): + async with websockets.connect(uri) as websocket: + # print(websocket.is_alive()) + # print(websocket.isAlive()) + # print(websocket.close()) + await websocket.send(json.dumps(params)) + # res = await websocket.recv() + # print(res) + try: + while True: + res = await websocket.recv() + res = json.loads(res) + print(res) + if res['done'] == True: + websocket.close() + print('Procress to 100% Success!') + break + except Exception as e: + websocket.close() + + +# main +asyncio.get_event_loop().run_until_complete(hello('ws://localhost:8101')) + +# while True: +# asyncio.get_event_loop().run_until_complete(hello('ws://localhost:8765')) From 0ffd0e06f1ac3f5f87cc16418182f241870a6fce Mon Sep 17 00:00:00 2001 From: JerryYin Date: Thu, 29 Nov 2018 18:32:31 +0800 Subject: [PATCH 20/42] basic done copy class's dataset in websocket --- .idea/workspace.xml | 452 +++++++----------- logs/NotebookServer-2018.11.29-13:39:56.log | 33 ++ logs/NotebookServer-2018.11.29-13:42:01.log | 17 + logs/NotebookServer-2018.11.29-13:42:07.log | 18 + logs/NotebookServer-2018.11.29-13:42:16.log | 18 + logs/NotebookServer-2018.11.29-16:50:15.log | 33 ++ logs/NotebookServer-2018.11.29-16:51:00.log | 46 ++ logs/NotebookServer-2018.11.29-16:53:59.log | 31 ++ logs/NotebookServer-2018.11.29-16:55:27.log | 32 ++ logs/NotebookServer-2018.11.29-16:56:26.log | 32 ++ logs/NotebookServer-2018.11.29-16:57:23.log | 32 ++ logs/NotebookServer-2018.11.29-16:59:42.log | 77 +++ logs/NotebookServer-2018.11.29-17:35:34.log | 34 ++ logs/NotebookServer-2018.11.29-17:44:18.log | 50 ++ logs/NotebookServer-2018.11.29-17:49:36.log | 34 ++ logs/NotebookServer-2018.11.29-17:51:12.log | 34 ++ logs/NotebookServer-2018.11.29-17:56:28.log | 34 ++ logs/NotebookServer-2018.11.29-17:57:23.log | 34 ++ logs/NotebookServer-2018.11.29-17:57:47.log | 34 ++ logs/NotebookServer-2018.11.29-17:58:03.log | 34 ++ logs/NotebookServer-2018.11.29-17:59:10.log | 34 ++ logs/NotebookServer-2018.11.29-17:59:42.log | 34 ++ logs/NotebookServer-2018.11.29-18:00:34.log | 34 ++ logs/NotebookServer-2018.11.29-18:01:13.log | 50 ++ logs/NotebookServer-2018.11.29-18:03:29.log | 57 +++ logs/NotebookServer-2018.11.29-18:12:34.log | 17 + logs/NotebookServer-2018.11.29-18:12:47.log | 34 ++ logs/NotebookServer-2018.11.29-18:13:38.log | 34 ++ logs/NotebookServer-2018.11.29-18:14:10.log | 50 ++ logs/NotebookServer-2018.11.29-18:18:46.log | 34 ++ logs/NotebookServer-2018.11.29-18:23:06.log | 34 ++ logs/NotebookServer-2018.11.29-18:24:20.log | 34 ++ logs/NotebookServer-2018.11.29-18:25:43.log | 40 ++ logs/NotebookServer-2018.11.29-18:27:01.log | 34 ++ logs/NotebookServer-2018.11.29-18:29:24.log | 35 ++ logs/NotebookServer-2018.11.29-18:29:51.log | 40 ++ .../__pycache__/projectManager.cpython-36.pyc | Bin 7309 -> 10321 bytes project/main/manager/projectManager.py | 227 ++++++++- .../__pycache__/httpServer.cpython-36.pyc | Bin 8534 -> 8534 bytes .../websocketServer.cpython-36.pyc | Bin 2370 -> 3241 bytes project/main/server/websocketServer.py | 122 ++++- project/test/__init__.py | 10 +- project/test/process.py | 21 + project/test/syncTask.py | 14 + project/test/wsClient.py | 28 +- 45 files changed, 1800 insertions(+), 326 deletions(-) create mode 100644 logs/NotebookServer-2018.11.29-13:39:56.log create mode 100644 logs/NotebookServer-2018.11.29-13:42:01.log create mode 100644 logs/NotebookServer-2018.11.29-13:42:07.log create mode 100644 logs/NotebookServer-2018.11.29-13:42:16.log create mode 100644 logs/NotebookServer-2018.11.29-16:50:15.log create mode 100644 logs/NotebookServer-2018.11.29-16:51:00.log create mode 100644 logs/NotebookServer-2018.11.29-16:53:59.log create mode 100644 logs/NotebookServer-2018.11.29-16:55:27.log create mode 100644 logs/NotebookServer-2018.11.29-16:56:26.log create mode 100644 logs/NotebookServer-2018.11.29-16:57:23.log create mode 100644 logs/NotebookServer-2018.11.29-16:59:42.log create mode 100644 logs/NotebookServer-2018.11.29-17:35:34.log create mode 100644 logs/NotebookServer-2018.11.29-17:44:18.log create mode 100644 logs/NotebookServer-2018.11.29-17:49:36.log create mode 100644 logs/NotebookServer-2018.11.29-17:51:12.log create mode 100644 logs/NotebookServer-2018.11.29-17:56:28.log create mode 100644 logs/NotebookServer-2018.11.29-17:57:23.log create mode 100644 logs/NotebookServer-2018.11.29-17:57:47.log create mode 100644 logs/NotebookServer-2018.11.29-17:58:03.log create mode 100644 logs/NotebookServer-2018.11.29-17:59:10.log create mode 100644 logs/NotebookServer-2018.11.29-17:59:42.log create mode 100644 logs/NotebookServer-2018.11.29-18:00:34.log create mode 100644 logs/NotebookServer-2018.11.29-18:01:13.log create mode 100644 logs/NotebookServer-2018.11.29-18:03:29.log create mode 100644 logs/NotebookServer-2018.11.29-18:12:34.log create mode 100644 logs/NotebookServer-2018.11.29-18:12:47.log create mode 100644 logs/NotebookServer-2018.11.29-18:13:38.log create mode 100644 logs/NotebookServer-2018.11.29-18:14:10.log create mode 100644 logs/NotebookServer-2018.11.29-18:18:46.log create mode 100644 logs/NotebookServer-2018.11.29-18:23:06.log create mode 100644 logs/NotebookServer-2018.11.29-18:24:20.log create mode 100644 logs/NotebookServer-2018.11.29-18:25:43.log create mode 100644 logs/NotebookServer-2018.11.29-18:27:01.log create mode 100644 logs/NotebookServer-2018.11.29-18:29:24.log create mode 100644 logs/NotebookServer-2018.11.29-18:29:51.log create mode 100644 project/test/process.py create mode 100644 project/test/syncTask.py diff --git a/.idea/workspace.xml b/.idea/workspace.xml index 625cefb..b195a8c 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -2,15 +2,13 @@ - - - + + - - - - + + + - @@ -328,7 +302,6 @@ - @@ -346,12 +319,6 @@ - - - - - - @@ -401,6 +368,7 @@ + + + + @@ -393,6 +395,13 @@ + + + + + + + @@ -403,11 +412,18 @@ - - \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml index 7307e9e..3df15c2 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -2,7 +2,10 @@ - + + + + @@ -322,7 +338,6 @@ - @@ -340,62 +355,11 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +