Merge pull request #2299 from avinashkranjan/deepsource-transform-baa3d8e1

format code with autopep8

Author: avinashkranjan

audio_detect/webserver/audio/encoder/encode.py

@@ -19,4 +19,4 @@ def get_audio_embedding(path):
     except Exception as e:
         print("error with embedding:", path)
         write_log(e, 1)
-        return path, None
+        return path, None

audio_detect/webserver/audio/indexer/index.py

@@ -18,7 +18,7 @@ def create_table_milvus(client, table_name, dimension):
         collection_param = {
             'collection_name': table_name,
             'dimension': dimension,
-            'index_file_size':2048,
+            'index_file_size': 2048,
             'metric_type':  METRIC_TYPE
         }
         status = client.create_collection(collection_param)
@@ -61,7 +61,8 @@ def delete_collection(client, table_name):
 def search_vectors(client, table_name, vectors, metric, top_k):
     search_param = {'nprobe': 32}
     try:
-        status, res = client.search(collection_name=table_name, query_records=vectors, top_k=top_k, params=search_param)
+        status, res = client.search(
+            collection_name=table_name, query_records=vectors, top_k=top_k, params=search_param)
         print(status)
         return res
     except Exception as e:
@@ -84,4 +85,4 @@ def count_collection(client, table_name):
         return num
     except Exception as e:
         print("Milvus ERROR:", e)
-        logging.error(e)
+        logging.error(e)

audio_detect/webserver/audio/indexer/logs.py

@@ -9,32 +9,34 @@
 except ImportError:
     codecs = None
 
+
 class MultiprocessHandler(logging.FileHandler):
-    def __init__(self,filename,when='D',backupCount=0,encoding=None,delay=False):
+    def __init__(self, filename, when='D', backupCount=0, encoding=None, delay=False):
 
         self.prefix = filename
         self.backupCount = backupCount
         self.when = when.upper()
         self.extMath = r"^\d{4}-\d{2}-\d{2}"
 
         self.when_dict = {
-            'S':"%Y-%m-%d-%H-%M-%S",
-            'M':"%Y-%m-%d-%H-%M",
-            'H':"%Y-%m-%d-%H",
-            'D':"%Y-%m-%d"
+            'S': "%Y-%m-%d-%H-%M-%S",
+            'M': "%Y-%m-%d-%H-%M",
+            'H': "%Y-%m-%d-%H",
+            'D': "%Y-%m-%d"
         }
 
         self.suffix = self.when_dict.get(when)
         if not self.suffix:
             raise ValueError(u"指定的日期间隔单位无效: %s" % self.when)
-        
-        self.filefmt = os.path.join("logs","%s.%s" % (self.prefix,self.suffix))
-        
+
+        self.filefmt = os.path.join("logs", "%s.%s" %
+                                    (self.prefix, self.suffix))
+
         self.filePath = datetime.datetime.now().strftime(self.filefmt)
-        
+
         _dir = os.path.dirname(self.filefmt)
         try:
-        
+
             if not os.path.exists(_dir):
                 os.makedirs(_dir)
         except Exception:
@@ -45,29 +47,30 @@ def __init__(self,filename,when='D',backupCount=0,encoding=None,delay=False):
         if codecs is None:
             encoding = None
 
-        logging.FileHandler.__init__(self,self.filePath,'a+',encoding,delay)
+        logging.FileHandler.__init__(
+            self, self.filePath, 'a+', encoding, delay)
 
     def shouldChangeFileToWrite(self):
-      
+
         _filePath = datetime.datetime.now().strftime(self.filefmt)
-      
+
         if _filePath != self.filePath:
             self.filePath = _filePath
             return True
         return False
 
     def doChangeFile(self):
-    
+
         self.baseFilename = os.path.abspath(self.filePath)
-  
+
         if self.stream:
-            
+
             self.stream.close()
-            
+
             self.stream = None
 
         if not self.delay:
-          
+
             self.stream = self._open()
 
         if self.backupCount > 0:
@@ -78,7 +81,7 @@ def doChangeFile(self):
 
     def getFilesToDelete(self):
 
-        dirName,_ = os.path.split(self.baseFilename)
+        dirName, _ = os.path.split(self.baseFilename)
         fileNames = os.listdir(dirName)
         result = []
 
@@ -90,10 +93,9 @@ def getFilesToDelete(self):
                 suffix = fileName[plen:]
 
                 if re.compile(self.extMath).match(suffix):
-                    result.append(os.path.join(dirName,fileName))
+                    result.append(os.path.join(dirName, fileName))
         result.sort()
 
-
         if len(result) < self.backupCount:
             result = []
         else:
@@ -105,8 +107,8 @@ def emit(self, record):
         try:
             if self.shouldChangeFileToWrite():
                 self.doChangeFile()
-            logging.FileHandler.emit(self,record)
-        except (KeyboardInterrupt,SystemExit):
+            logging.FileHandler.emit(self, record)
+        except (KeyboardInterrupt, SystemExit):
             raise
         except:
             self.handleError(record)
@@ -121,18 +123,14 @@ def write_log(log_message, level=0):
     stream_handler.setLevel(logging.INFO)
     stream_handler.setFormatter(fmt)
 
-
     log_name = "app.log"
     file_handler = MultiprocessHandler(log_name, when='D', backupCount=7)
     file_handler.setLevel(logging.INFO)
     file_handler.setFormatter(fmt)
 
-
     logger.addHandler(stream_handler)
     logger.addHandler(file_handler)
     if level:
         logger.error(log_message)
     else:
         logger.info(log_message)
-
-

audio_detect/webserver/audio/indexer/tools.py

@@ -8,27 +8,31 @@
 def connect_mysql():
     try:
         # conn = pymysql.connect(host="127.0.0.1",user="root",port=3306,password="123456",database="mysql", local_infile=True)
-        conn = pymysql.connect(host=MYSQL_HOST,user=MYSQL_USER,port=MYSQL_PORT,password=MYSQL_PWD,database=MYSQL_DB, local_infile=True)
+        conn = pymysql.connect(host=MYSQL_HOST, user=MYSQL_USER, port=MYSQL_PORT,
+                               password=MYSQL_PWD, database=MYSQL_DB, local_infile=True)
         return conn
     except Exception as e:
         print("MYSQL ERROR: connect failed", e)
-        write_log(e,1)
+        write_log(e, 1)
 
 
-def create_table_mysql(conn,cursor, table_name):
-    sql = "create table if not exists " + table_name + "(milvus_id bigint, audio_id text, index ix_milvus (milvus_id));"
+def create_table_mysql(conn, cursor, table_name):
+    sql = "create table if not exists " + table_name + \
+        "(milvus_id bigint, audio_id text, index ix_milvus (milvus_id));"
     try:
         cursor.execute(sql)
         print("MYSQL create table.")
     except Exception as e:
         print("MYSQL ERROR:", sql, e)
-        write_log(e,1)
+        write_log(e, 1)
 
 
 def search_by_milvus_ids(conn, cursor, ids, table_name):
     str_ids = str(ids)
-    str_ids = str(str_ids).replace('[','').replace(']','')
-    sql = "select audio_id from " + table_name + " where milvus_id in (" + str_ids + ") order by field (milvus_id," + str_ids + ");"
+    str_ids = str(str_ids).replace('[', '').replace(']', '')
+    sql = "select audio_id from " + table_name + \
+        " where milvus_id in (" + str_ids + \
+        ") order by field (milvus_id," + str_ids + ");"
     # print(sql)
     try:
         cursor.execute(sql)
@@ -38,24 +42,26 @@ def search_by_milvus_ids(conn, cursor, ids, table_name):
         return results
     except Exception as e:
         print("MYSQL ERROR:", sql, e)
-        write_log(e,1)
+        write_log(e, 1)
 
 
 def load_data_to_mysql(conn, cursor, table_name, file_name):
-    sql = "load data local infile '" + file_name + "' into table " + table_name + " fields terminated by ',';"
+    sql = "load data local infile '" + file_name + \
+        "' into table " + table_name + " fields terminated by ',';"
     try:
         cursor.execute(sql)
         conn.commit()
         print("MYSQL load table.")
     except Exception as e:
         print("MYSQL ERROR:", sql, e)
-        write_log(e,1)
+        write_log(e, 1)
     finally:
         if os.path.exists(file_name):
             with open(file_name) as f:
                 line = f.readlines()
                 print("-----------MySQL insert info--------len:" + str(len(line)))
-                write_log("-----------MySQL insert info--------len:" + str(len(line)))
+                write_log(
+                    "-----------MySQL insert info--------len:" + str(len(line)))
             os.remove(file_name)
 
 
@@ -66,7 +72,7 @@ def delete_table(conn, cursor, table_name):
         print("MYSQL delete table.")
     except:
         print("MYSQL ERROR:", sql, e)
-        write_log(e,1)
+        write_log(e, 1)
 
 
 def delete_all_data(conn, cursor, table_name):
@@ -77,7 +83,7 @@ def delete_all_data(conn, cursor, table_name):
         print("MYSQL delete all data.")
     except:
         print("MYSQL ERROR:", sql, e)
-        write_log(e,1)
+        write_log(e, 1)
 
 
 def count_table(conn, cursor, table_name):
@@ -89,4 +95,4 @@ def count_table(conn, cursor, table_name):
         return results[0][0]
     except Exception as e:
         print("MYSQL ERROR:", sql, e)
-        write_log(e,1)
+        write_log(e, 1)

audio_detect/webserver/audio/service/count.py

@@ -5,6 +5,7 @@
 from audio.indexer.index import milvus_client, count_collection
 from audio.indexer.tools import count_table
 
+
 def do_count_table(index_client, conn, cursor, table_name):
     if not table_name:
         table_name = DEFAULT_TABLE

audio_detect/webserver/audio/service/insert.py

@@ -19,7 +19,7 @@ def init_table(index_client, conn, cursor, table_name):
 
 
 def get_ids_file(ids_milvus, ids_audio, file_name):
-    with open(file_name,'w') as f:
+    with open(file_name, 'w') as f:
         for i in range(len(ids_audio)):
             line = str(ids_milvus[i]) + "," + ids_audio[i] + '\n'
             f.write(line)
@@ -48,7 +48,7 @@ def get_wav_info(audio_path, wav):
 
 
 def do_insert_audio(index_client, conn, cursor, table_name, audio_path):
-    
+
     try:
         init_table(index_client, conn, cursor, table_name)
         wavs = os.listdir(audio_path)
@@ -58,14 +58,15 @@ def do_insert_audio(index_client, conn, cursor, table_name, audio_path):
         for wav in wavs:
             # print("---wav:", wav)
             if ".wav" in wav:
-                ids_wav, vectors_wav = get_audio_embedding(audio_path + '/' + wav)
+                ids_wav, vectors_wav = get_audio_embedding(
+                    audio_path + '/' + wav)
                 if vectors_wav:
                     get_spectorgram(audio_path, wav)
                     embeddings.append(vectors_wav)
                     ids_audio.append(ids_wav)
                 # print("len of embeddings", len(embeddings))
         _, ids_milvus = insert_vectors(index_client, table_name, embeddings)
-        
+
         file_name = str(uuid.uuid1()) + ".csv"
         get_ids_file(ids_milvus, ids_audio, file_name)
         print("load data to mysql:", file_name)

audio_detect/webserver/audio/service/search.py

@@ -13,7 +13,8 @@ def do_search_audio(index_client, conn, cursor, table_name, filename):
     _, vectors_audio = get_audio_embedding(filename)
     # print(vectors_audio)
 
-    results = search_vectors(index_client, table_name, [vectors_audio], METRIC_TYPE, TOP_K)
+    results = search_vectors(index_client, table_name, [
+                             vectors_audio], METRIC_TYPE, TOP_K)
 
     re_ids_img = []
     milvus_ids = [x.id for x in results[0]]

audio_detect/webserver/main.py

@@ -54,11 +54,12 @@ def unzip_file(zip_src, dst_dir):
 async def audio_count_table_api(table_name: str = None):
     try:
         index_client, conn, cursor = audio_init_conn()
-        rows_milvus, rows_mysql = audio_count_table(index_client, conn, cursor, table_name)
+        rows_milvus, rows_mysql = audio_count_table(
+            index_client, conn, cursor, table_name)
         return {'status': True, 'msg': {'rows_milvus': rows_milvus, 'rows_mysql': rows_mysql}}, 200
     except Exception as e:
         logging.error(e)
-        return {'status': False, 'msg':e}, 400
+        return {'status': False, 'msg': e}, 400
 
 
 @app.get('/getAudio')
@@ -68,7 +69,7 @@ async def audio_endpoint(audio: str):
         return FileResponse(audio)
     except Exception as e:
         logging.error(e)
-        return {'status': False, 'msg':e}, 400
+        return {'status': False, 'msg': e}, 400
 
 
 @app.get('/getSpectrogram')
@@ -78,7 +79,7 @@ async def spectrogram_endpoint(image: str):
         return FileResponse(image)
     except Exception as e:
         logging.error(e)
-        return {'status': False, 'msg':e}, 400
+        return {'status': False, 'msg': e}, 400
 
 
 @app.post('/audio/insert')
@@ -91,17 +92,19 @@ async def do_insert_audio_api(file: bytes = File(...), table_name: str = None):
 
         os.makedirs(audio_UPLOAD_PATH + "/" + table_name)
         fname_path = audio_UPLOAD_PATH + "/" + table_name + "/" + "demo_audio.zip"
-        with open(fname_path,'wb') as f:
+        with open(fname_path, 'wb') as f:
             f.write(file)
 
-        audio_path = unzip_file(fname_path, audio_UPLOAD_PATH + "/" + table_name)
+        audio_path = unzip_file(
+            fname_path, audio_UPLOAD_PATH + "/" + table_name)
         os.remove(fname_path)
 
-        info = audio_insert_audio(index_client, conn, cursor, table_name, audio_UPLOAD_PATH + "/" + table_name)
+        info = audio_insert_audio(
+            index_client, conn, cursor, table_name, audio_UPLOAD_PATH + "/" + table_name)
         return {'status': True, 'msg': info}, 200
     except Exception as e:
         logging.error(e)
-        return {'status': False, 'msg':e}, 400
+        return {'status': False, 'msg': e}, 400
 
 
 @app.post('/audio/search')
@@ -114,8 +117,9 @@ async def do_search_audio_api(request: Request, audio: UploadFile = File(...), t
 
         index_client, conn, cursor = audio_init_conn()
         host = request.headers['host']
-        milvus_ids, milvus_distance, audio_ids = audio_search_audio(index_client, conn, cursor, table_name, filename)
-        
+        milvus_ids, milvus_distance, audio_ids = audio_search_audio(
+            index_client, conn, cursor, table_name, filename)
+
         results = []
         for i in range(len(milvus_ids)):
             re = {
@@ -128,8 +132,7 @@ async def do_search_audio_api(request: Request, audio: UploadFile = File(...), t
         return {'status': True, 'msg': results}, 200
     except Exception as e:
         logging.error(e)
-        return {'status': False, 'msg':e}, 400
-
+        return {'status': False, 'msg': e}, 400
 
 
 if __name__ == '__main__':