@ -1,5 +1,7 @@
# Copyright (c) 2021, Hard n Soft Technologies Pvt Ltd and contributors
# For license information, please see license.txt
from deepdiff import DeepDiff
import pandas as pd
import copy
import sys
from html import unescape
@ -13,7 +15,6 @@ from frappe.utils import logger
current_db_name = frappe . conf . get ( " db_name " )
sys . tracebacklimit = 0
site_name = cstr ( frappe . local . site )
import pandas as pd
base_url = os . path . expanduser (
" ~ " ) + " /frappe-bench/sites/ " + site_name + " /public " + " /files " + " /json_files " + " /phase2 "
@ -22,12 +23,15 @@ frappe.utils.logger.set_log_level("DEBUG")
success_reponse = { " status " : 1 , " data " : " " , " message " : " " }
failure_reponse = { " status " : 0 , " data " : " " , " error " : " " }
module_name = ' feature_finder '
exisitng_var = [ ]
new_variant = [ ]
class Publish ( Document ) :
def validate ( self ) :
self . published_date = str ( date . today ( ) )
self . published_by = frappe . session . user
# Selected variant and kilometer mapping details list
variant = [ ]
for vm in self . get ( ' publish_documents ' ) :
@ -37,10 +41,10 @@ class Publish(Document):
# variant.append(vm.variant_mapping)
if vm . variant_mapping not in variant :
variant . append ( vm . variant_mapping )
if self . publish_module == ' Feature Finder ' and self . feature_finder_publish_docs :
if self . publish_module == ' Feature Finder ' and self . feature_finder_publish_docs :
for vm in self . get ( ' feature_finder_publish_docs ' ) :
if vm . variant not in variant :
variant . append ( vm . variant )
if vm . variant not in variant :
variant . append ( vm . variant )
self . variant_mapping_details = ' '
self . kilometer_mapping_details = ' '
@ -85,7 +89,7 @@ class Publish(Document):
update_repair_published_docs ( self )
update_publish_status = frappe . db . sql (
""" update `tabPublish` set publish_status= ' Published ' where name= ' {0} ' """ . format ( self . name ) )
variant = self . variant_mapping_details . split ( ' /n ' )
frappe . log_error ( ' variant ' , str ( variant ) )
@ -95,56 +99,6 @@ class Publish(Document):
# update_publish_mapping(
# self.vehicle, v, self.language, self.publish_module)
def onload1 ( self ) :
current_db_name = frappe . conf . get ( " db_name " )
# To validate updated procedure status
procedure_status = frappe . db . sql ( """ select p.name,p.update_procedure_status,pro.name,p.variant_mapping,p.system,p.sub_system,p.asset_category,
p . item_category , p . variant , p . procedure_link , p . procedure_status , pro . name , pro . workflow_state
from { 0 } . ` tabPublish_Docs ` as p , { 0 } . ` tabProcedure ` as pro where p . parent = ' {1} ' and
p . procedure_link = pro . name and p . procedure_link is not null """ .format(current_db_name, self.name), as_dict=1)
l = len ( procedure_status )
if procedure_status and self . publish_status == " Published " :
for x in procedure_status :
for y in self . get ( " publish_documents " ) :
if y . procedure_link :
if x . procedure_link == y . procedure_link and y . update_procedure_status != y . procedure_status :
y . update_procedure_status = x . workflow_state
frappe . db . sql ( """ update `tabPublish_Docs` set update_procedure_status = %s where parent = %s and procedure_link = %s ; """ , (
x . workflow_state , self . name , x . procedure_link ) )
frappe . db . commit ( )
if x . procedure_link == y . procedure_link and x . workflow_state != y . procedure_status :
y . update_procedure_status = x . workflow_state
frappe . db . sql ( """ update `tabPublish_Docs` set update_procedure_status = %s where parent = %s and procedure_link = %s """ , (
x . workflow_state , self . name , x . procedure_link ) )
frappe . db . commit ( )
if y . procedure_status == " Publish Ready " :
if x . procedure_link == y . procedure_link and x . workflow_state != y . procedure_status and not y . excluded_global :
y . excluded_global = " 1 "
if x . procedure_link == y . procedure_link and x . workflow_state != y . procedure_status and x . workflow_state == " Draft " and y . procedure_status != " Publish Ready " :
y . excluded_internal = " 1 "
y . save ( )
frappe . db . commit ( )
# Update system and variant mapping ststus
if self . publish_status == " Published " and self . publish_type == " Global " :
current_db_name = frappe . conf . get ( " db_name " )
global_publish = frappe . db . sql ( """ select name from {0} .`tabPublish` where global_publish = " {1} " ; """ . format (
current_db_name , self . name ) , as_dict = True )
for global_name in global_publish :
update_mapping ( self . variant_mapping , global_name . name )
u_id = frappe . get_all (
" User " , filters = { " name " : frappe . session . user } , fields = [ " username " ] )
if u_id :
for i in u_id :
name = i . username
if not self . published_by and self . publish_status == " Published " :
self . published_by = str ( name )
frappe . db . set_value (
" Publish " , self . name , " published_by " , str ( name ) )
frappe . db . commit ( )
def on_cancel ( self ) :
# Published document should not allow to cancel
if self . publish_status == " Published " :
@ -204,25 +158,26 @@ def update_procedure(vehicle, lang, publish_type, doc):
# Generate Publish versions
def update_publish_mapping ( vehicle , variant , language , module , publish_type ) :
def update_publish_mapping ( vehicle , variant , language , module , publish_type ) :
frappe . set_user ( ' Administrator ' )
if module == ' Repair service ' :
pub_data = frappe . db . get_list ( ' Module Publish Mapping ' , filters = {
" vehicle " : vehicle ,
" language " : language ,
" publish_type " : publish_type
" publish_type " : publish_type
} , fields = [ ' name ' ] )
if len ( pub_data ) > 0 :
if len ( pub_data ) > 0 :
for d in pub_data :
if d [ ' name ' ] :
frappe . db . sql ( f """ UPDATE `tabModule Publish Mapping` set repairservice_check_sheet= ' 1 ' and publish_type= ' 1 ' where name = ' { d [ ' name ' ] } ' """ , as_dict = 1 )
frappe . db . sql (
f """ UPDATE `tabModule Publish Mapping` set repairservice_check_sheet= ' 1 ' and publish_type= ' 1 ' where name = ' { d [ ' name ' ] } ' """ , as_dict = 1 )
frappe . db . commit ( )
else :
pub_data = frappe . db . get_list ( ' Module Publish Mapping ' , filters = {
" vehicle " : vehicle ,
" variant " : variant ,
" language " : language ,
" publish_type " : publish_type
" publish_type " : publish_type
} , fields = [ ' name ' ] )
if pub_data :
@ -255,12 +210,18 @@ def update_publish_mapping(vehicle, variant, language, module,publish_type):
@frappe . whitelist ( )
def cal_ver ( vehicle , lang , publish_type , doc ) :
def cal_ver ( vehicle , lang , publish_type , doc , publish_module ) :
update_procedure ( vehicle , lang , publish_type , doc )
current_db_name = frappe . conf . get ( " db_name " )
doc = frappe . get_doc ( " Publish " , doc )
ver = frappe . db . sql ( """ select max(version) from `tabPublish` where vehicle = " {vehi} " and language = " {lang} " ; """
. format ( vehi = vehicle , lang = lang ) )
if publish_module == ' Automotive System ' :
ver = frappe . db . sql ( """ select max(version) from `tabPublish` where vehicle = " {vehi} " and language = " {lang} "
and publish_module = ' Automotive System ' ; """
. format ( vehi = vehicle , lang = lang ) )
else :
ver = frappe . db . sql ( """ select max(version) from `tabPublish` where vehicle = " {vehi} " and language = " {lang} "
and publish_module != ' Automotive System ' ; """
. format ( vehi = vehicle , lang = lang ) )
v = 0.0
if publish_type == ' Global ' :
if ver [ 0 ] [ 0 ] == None :
@ -334,28 +295,28 @@ def global_publish(doc):
@frappe . whitelist ( )
def generate_global_publish ( name , module_name ) :
def generate_global_publish ( name , module_name ) :
try :
res = frappe . get_doc ( " Publish " , name )
if module_name == ' Feature Finder ' :
if module_name == ' Feature Finder ' :
ret = frappe . get_doc ( {
" doctype " : " Publish " ,
" vehicle " : res . vehicle ,
" publish_type " : " Global " ,
" language " : res . language ,
" variant_mapping_details " : res . variant_mapping_details ,
" variant_mapping_details " : res . variant_mapping_details ,
" publish_status " : ' To Publish ' ,
" release_description " : res . release_description ,
" publish_module " : res . publish_module ,
" feature_finder_publish_docs " : res . feature_finder_publish_docs
} )
elif module_name == ' Repair service ' :
ret = frappe . get_doc ( {
ret = frappe . get_doc ( {
" doctype " : " Publish " ,
" vehicle " : res . vehicle ,
" publish_type " : " Global " ,
" language " : res . language ,
" variant_mapping_details " : res . variant_mapping ,
" variant_mapping_details " : res . variant_mapping ,
" publish_status " : ' To Publish ' ,
" release_description " : res . release_description ,
" publish_module " : res . publish_module ,
@ -583,19 +544,22 @@ def get_service_repair(vehicle,
select * from ` tabRepair Service Mapping ` where vehicle = ' {vehicle} ' and language = ' {language_label} ' and published = 0 ;
''' , as_dict=1)
return data
def create_df ( data_set ) :
feature_finder_tmp = json . dumps ( data_set )
feature_finder_tmp = json . dumps ( data_set )
df = pd . DataFrame ( json . loads ( feature_finder_tmp ) )
filter_group = df . groupby ( ' variant ' )
keys = filter_group . groups . keys ( )
res = { }
filter_group = df . groupby ( ' variant ' )
keys = filter_group . groups . keys ( )
res = { }
for i in keys :
res [ i ] = ( filter_group . get_group ( i ) )
res1 = { }
for key , value in res . items ( ) :
res1 [ key ] = value . to_dict ( ' records ' )
res [ i ] = ( filter_group . get_group ( i ) )
res1 = { }
for key , value in res . items ( ) :
res1 [ key ] = value . to_dict ( ' records ' )
return res1
@frappe . whitelist ( )
def get_feature_finder ( vehicle = None , variant = None , language_label = None ) :
try :
@ -715,13 +679,15 @@ def feature_finder_publish(vehicle=None, vehicle_id=None,
create_publish_folders ( folder_url )
file_path = folder_url + " / " + publish_type + " / " + \
vehicle . replace ( ' ' , ' - ' ) + ' -feature_finder ' + ' .json '
global_file_path = folder_url + " / " + " Global " + " / " + \
global_file_path = folder_url + " / " + ' Global ' + " / " + \
vehicle . replace ( ' ' , ' - ' ) + ' -feature_finder ' + ' .json '
''' Append Published Data to Json '''
logger . info (
f ' Repair Checksheet Data Append Start:: { vehicle } - { language } - { publish_type } ' )
''' update existing global json file '''
if os . path . isfile ( file_path ) and publish_type == ' Internal ' :
if os . path . isfile ( global_file_path ) and publish_type == ' Internal ' :
with open ( global_file_path ) as f :
published_data = json . load ( f )
for i in parent :
@ -731,21 +697,36 @@ def feature_finder_publish(vehicle=None, vehicle_id=None,
publish_repair_checksheet = 1
feature_finder_tmp . append (
feature_finder [ ' data ' ] [ 0 ] )
feature_finder_tmp = create_df ( feature_finder_tmp )
vehi_data = compare_get_data ( { ' data ' : published_data [ ' data ' ] } , { ' data ' : feature_finder_tmp } )
feature_finder_tmp = create_df ( feature_finder_tmp )
vehi_data = compare_get_data ( { ' data ' : published_data [ ' data ' ] } , {
' data ' : feature_finder_tmp } )
if vehi_data :
find_distinct = set ( exisitng_var )
new_variant_name = [
x for x in new_variant if x not in find_distinct ]
vehi_data = add_new_val (
vehi_data , new_variant_name , feature_finder_tmp )
elif os . path . isfile ( file_path ) and publish_type == ' Global ' :
frappe . msgprint ( str ( ' called ' ) )
with open ( global_file_path ) as f :
published_data = json . load ( f )
for i in parent :
feature_finder = feature_finder_data ( vehicle , language ,
publish_type , i . parent1 , i . variant )
publish_type , i . parent1 , i . variant )
if feature_finder [ ' status ' ] == 1 :
publish_repair_checksheet = 1
feature_finder_tmp . append (
feature_finder [ ' data ' ] [ 0 ] )
feature_finder_tmp = create_df ( feature_finder_tmp )
vehi_data = compare_get_data ( { ' data ' : published_data [ ' data ' ] } , { ' data ' : feature_finder_tmp } )
feature_finder_tmp = create_df ( feature_finder_tmp )
vehi_data = compare_get_data ( { ' data ' : published_data [ ' data ' ] } , {
' data ' : feature_finder_tmp } )
if vehi_data :
find_distinct = set ( exisitng_var )
new_variant_name = [
x for x in new_variant if x not in find_distinct ]
vehi_data = add_new_val (
vehi_data , new_variant_name , feature_finder_tmp )
else :
for i in parent :
feature_finder = feature_finder_data (
@ -761,8 +742,8 @@ def feature_finder_publish(vehicle=None, vehicle_id=None,
feature_finder_tmp = get_latest_data (
{ ' data ' : feature_finder_tmp } , { ' data ' : [ ] } )
vehi_data = create_df ( feature_finder_tmp )
vehi_data = create_df ( feature_finder_tmp )
""" Save publish file """
vehicle_data [ ' data ' ] = vehi_data
@ -803,11 +784,12 @@ def get_latest_data(prev_data, latest_data):
final_dic . update ( { d2 : data_dic2 [ d2 ] } )
final_list = list ( final_dic . values ( ) )
return final_list
except Exception as e :
frappe . throw ( str ( e ) )
def compare_get_data ( prev_data , latest_data ) :
try :
logger_file = ' compare '
@ -815,60 +797,89 @@ def compare_get_data(prev_data, latest_data):
allow_site = True , file_count = 100 )
logger . info (
f " start compare data " )
final_dic = { }
final_dic = prev_data . copy ( )
for key , value in final_dic [ " data " ] . items ( ) :
for i in value :
if i [ ' active_status ' ] == " Inactive " :
value . remove ( i )
for key , value in latest_data [ " data " ] . items ( ) :
for i in value :
if i [ ' active_status ' ] == " Inactive " :
value . remove ( i )
#compare old & new json
for key , values in latest_data [ " data " ] . items ( ) :
final_dic = { }
final_dic = prev_data . copy ( )
# for key,value in latest_data["data"].items():
# for i in value:
# if i['active_status']=="Inactive":
# value.remove(i)
# compare old & new json
for key , values in latest_data [ " data " ] . items ( ) :
for i in latest_data [ " data " ] [ key ] :
check_key ( { key : i } , final_dic [ " data " ] , ' ' )
check_key ( { key : i } , final_dic [ " data " ] , ' ' )
# for key,value in final_dic["data"].items():
# for i in value:
# if i['active_status']=="Inactive":
# value.remove(i)
logger . info (
f " End of compare data " + str ( final_dic [ ' data ' ] ) )
return final_dic [ ' data ' ]
except Exception as e :
logger . error ( str ( e ) )
frappe . throw ( str ( e ) )
def check_key ( key_name , old_data , type = None ) :
try :
frappe . msgprint ( str ( key_name ) )
logger_file = ' check key '
logger = frappe . logger ( logger_file ,
allow_site = True , file_count = 100 )
logger . info (
f " start of key data " )
new_dict = [ ]
for key , val in key_name . items ( ) :
new_dict_lst = [ ]
new_dict = { }
for key , val in key_name . items ( ) :
if key in old_data :
if val [ ' name ' ] in old_data :
for old_key , old_val in old_data . items ( ) :
for i in old_val :
if val [ ' name ' ] in old_val :
if old_key == key and val [ ' name ' ] == i [ ' name ' ] :
i . update ( val )
else :
old_data [ key ] . append ( val )
for old_key , old_val in old_data . items ( ) :
for i in old_val :
if val [ ' name ' ] in i . values ( ) :
exisitng_var . append ( val [ ' name ' ] )
cmp = DeepDiff ( i , val )
if cmp . get ( ' values_changed ' ) :
indx = old_data [ key ] . index ( i )
old_data [ key ] [ indx ] = val
else :
indx = old_data [ key ] . index ( i )
old_data [ key ] [ indx ] = val
else :
if val [ ' name ' ] not in new_variant :
new_variant . append ( val [ ' name ' ] )
else :
new_dict . append ( val )
old_data [ key ] = new_dict
new_dict . update ( val )
new_dict_lst . append ( new_dict )
old_data [ key ] = new_dict_lst
# for key,val in key_name.items():
# if key in old_data:
# if val['name'] in old_data:
# for old_key,old_val in old_data.items():
# for i in old_val:
# if val['name'] in old_val :
# if old_key == key and val['name'] == i['name']:
# i.update(val)
# else:
# old_data[key].append(val)
# else:
# new_dict.append(val)
# old_data[key]=new_dict
logger . info ( f " end of key data " + str ( key_name ) )
except Exception as e :
logger . error ( str ( e ) )
return str ( e )
def add_new_val ( old_data , new_val_variant , dt_new ) :
for dt_key , dt_val in dt_new . items ( ) :
for i in dt_val :
if i [ ' name ' ] in new_val_variant :
old_data [ dt_key ] . append ( i )
return old_data
def repair_checksheet_data ( vehicle = None , language = None ,
publish_type = None , parent = None ) :
try :
@ -880,6 +891,7 @@ def repair_checksheet_data(vehicle=None, language=None,
repair_service_details = frappe . db . sql ( f """ select vehicle,vehicle_id,
name , language , check_list_name ,
keywords ,
active_status , display_order , my_id
from ` tabRepair Service Mapping `
where vehicle = ' {vehicle} '
@ -917,6 +929,7 @@ def feature_finder_data(vehicle=None, language=None,
feature_finder_details = [ ]
if language == ' en ' :
feature_finder_details = frappe . db . sql ( f """ select ff.name,ff.vehicle,ff.vehicle_id,ff.variant,ff.variant_id,ff.language,
ff . keywords ,
ff . features_names as feature_name , ff . feature_category , ff . display_order , ff . active_status ,
ffm . feature_pdf as pdf from ` tabFeature Finder ` ff left join ` tabFeature Finder Master ` ffm on
ff . features_names = ffm . feature_name
@ -927,6 +940,7 @@ def feature_finder_data(vehicle=None, language=None,
logger . info ( feature_finder_details )
else :
feature_finder_details = frappe . db . sql ( f """ select ff.name,ff.vehicle,ff.vehicle_id,ff.variant,ff.variant_id,ff.language,
ff . keywords ,
ff . translated_feature_name as feature_name , ff . feature_category ,
ff . display_order , ff . active_status , ffm . feature_pdf as pdf from ` tabFeature Finder ` ff
left join ` tabFeature Finder Master ` ffm on ff . features_names = ffm . feature_name
@ -995,7 +1009,8 @@ def max_publish_new_module(doc):
def cal_ver_new_module ( vehicle , lang , publish_type , doc ) :
current_db_name = frappe . conf . get ( " db_name " )
doc = frappe . get_doc ( " Publish " , doc )
ver = frappe . db . sql ( """ select max(version) from `tabPublish` where vehicle = " {vehi} " and language = " {lang} " and publish_module!= " Automotive System " ; """
ver = frappe . db . sql ( """ select max(version) from `tabPublish` where vehicle = " {vehi} " and language = " {lang} " and
publish_module != " Automotive System " ; """
. format ( vehi = vehicle , lang = lang ) )
v = 0.0
if publish_type == ' Global ' :
