#!/usr/bin/python3 # -*- coding:UTF-8 -*- # ===================================================================== # @module: Get all blx commits and save to csv file # @author: Li Dongqing (dongqing.li@amlogic.com) # @License: Copyright (c) 2019 Amlogic, Inc. All rights reserved. # @Changes: # 1. 2022.07.20 v0.1 create for stable branch update. # ===================================================================== # Import modules try: import sys import os import re import json import time import argparse import subprocess import IPython import openpyxl from git.repo import Repo from openpyxl.styles import Font, Alignment from collections import OrderedDict import ast except Exception as e: print(e) exit('Please install modules, eg: pip3 install os') # jason example: bl3.4.5-20220711-pre-ver.json # { # "source_gits" : [ # {"blType" : "bl2_sc2", "gitPath" : "bl2/core_sc2", "lastCommit" : "feebe5301418c038a06d45d0216c780ae9ea0033"}, # {"blType" : "bl2_s4", "gitPath" : "bl2/core_s4", "lastCommit" : "817779a738e99b81081a31035ed784840cace44c"}, # {"blType" : "bl2_ree", "gitPath" : "bl2/ree", "lastCommit" : "09d5c246638e95e2598a264b36da7f6ede7f6ea8"}, # {"blType" : "bl2_tee", "gitPath" : "bl2/tee", "lastCommit" : "115e9fc38721c3564435f582875fc02908787b53"}, # {"blType" : "bl30_aocpu", "gitPath" : "bl30/src_ao", "lastCommit" : "8c0b17692bd51f9c3311f8c51cd28bdf808a27a3"}, # {"blType" : "bl31_1.3", "gitPath" : "bl31_1.3/src", "lastCommit" : "cf6108ce548d7ad3bfe268dedf801358664b6ead"}, # {"blType" : "bl32_3.8", "gitPath" : "bl32_3.8/src", "lastCommit" : "c5ef42f2ce59304e2a4df7cf2dcbb12ab7ccefd1"}, # {"blType" : "bl33", "gitPath" : "bl33/v2019", "lastCommit" : "f03ed9bc121114e9f31f1ee924d3adc176f13faa"}, # {"blType" : "fip", "gitPath" : "fip", "lastCommit" : "d60ea7c9adfe8e537d0a11d2c2ce8e8097de5035"} # ] # } # bootloader trunk branch and remote info pdPrefix = "https://jira.amlogic.com/browse/" scgitPrefix = "https://scgit.amlogic.com/plugins/gitiles/" blSrcGits = [ {"blType" : "bl2_sc2", "gitBranch" : "projects/sc2", "stbBranch" : "projects/openlinux/bl-sc2-3.5.0", "gitRemote" : "firmware", "upStream" : "bootloader/amlogic-advanced-bootloader/core/+/"}, {"blType" : "bl2_s4", "gitBranch" : "projects/s4", "stbBranch" : "projects/openlinux/bl-s4-3.5.0", "gitRemote" : "firmware", "upStream" : "bootloader/amlogic-advanced-bootloader/core/+/"}, {"blType" : "bl2_s5", "gitBranch" : "projects/s5", "stbBranch" : "projects/openlinux/bl-s5-3.5.0", "gitRemote" : "firmware", "upStream" : "bootloader/amlogic-advanced-bootloader/core/+/"}, {"blType" : "bl2_t7", "gitBranch" : "projects/t7", "stbBranch" : "projects/openlinux/bl-t7-3.5.0", "gitRemote" : "firmware", "upStream" : "bootloader/amlogic-advanced-bootloader/core/+/"}, {"blType" : "bl2_s7", "gitBranch" : "projects/s7", "stbBranch" : "projects/openlinux/bl-s7-3.5.0", "gitRemote" : "firmware", "upStream" : "bootloader/amlogic-advanced-bootloader/core/+/"}, {"blType" : "bl2_s7d", "gitBranch" : "projects/amlogic-dev", "stbBranch" : "projects/openlinux/bl-s7d-3.5.0", "gitRemote" : "firmware", "upStream" : "bootloader/amlogic-advanced-bootloader/core/+/"}, {"blType" : "bl2_s6", "gitBranch" : "projects/s6", "stbBranch" : "projects/openlinux/bl-s6-3.5.0", "gitRemote" : "firmware", "upStream" : "bootloader/amlogic-advanced-bootloader/core/+/"}, {"blType" : "ddr", "gitBranch" : "amlogic-ddr-core", "stbBranch" : "projects/openlinux/bl-3.5.0", "gitRemote" : "firmware", "upStream" : "bl2/src/ddr/+/"}, {"blType" : "bl2_ree", "gitBranch" : "projects/amlogic-dev", "stbBranch" : "projects/openlinux/bl-3.5.0", "gitRemote" : "firmware", "upStream" : "bootloader/amlogic-advanced-bootloader/ree/+/"}, {"blType" : "bl2_tee", "gitBranch" : "projects/amlogic-dev", "stbBranch" : "projects/openlinux/bl-3.5.0", "gitRemote" : "firmware", "upStream" : "bootloader/amlogic-advanced-bootloader/tee/+/"}, {"blType" : "bl30_aocpu", "gitBranch" : "projects/amlogic-dev", "stbBranch" : "projects/openlinux/bl-3.5.0", "gitRemote" : "firmware", "upStream" : "firmware/aocpu/+/"}, {"blType" : "rtos_arch", "gitBranch" : "projects/amlogic-dev", "stbBranch" : "projects/openlinux/bl-3.5.0", "gitRemote" : "origin", "upStream" : "rtos_sdk/arch/riscv/+/"}, {"blType" : "rtosboards", "gitBranch" : "projects/amlogic-dev", "stbBranch" : "projects/openlinux/bl-3.5.0", "gitRemote" : "origin", "upStream" : "rtos_sdk/boards/riscv/+/"}, {"blType" : "rtos_build", "gitBranch" : "projects/amlogic-dev", "stbBranch" : "projects/openlinux/bl-3.5.0", "gitRemote" : "origin", "upStream" : "rtos_sdk/build/+/"}, {"blType" : "rtos_driver_aocpu", "gitBranch" : "projects/amlogic-dev", "stbBranch" : "projects/openlinux/bl-3.5.0", "gitRemote" : "origin", "upStream" : "rtos_sdk/drivers_aocpu/+/"}, {"blType" : "rtos_freertos", "gitBranch" : "projects/amlogic-dev", "stbBranch" : "projects/openlinux/bl-3.5.0", "gitRemote" : "origin", "upStream" : "rtos_sdk/freertos/+/"}, {"blType" : "rtos_libc", "gitBranch" : "projects/amlogic-dev", "stbBranch" : "projects/openlinux/bl-3.5.0", "gitRemote" : "origin", "upStream" : "rtos_sdk/libc/+/"}, {"blType" : "rtos_products_aocpu", "gitBranch" : "projects/amlogic-dev", "stbBranch" : "projects/openlinux/bl-3.5.0", "gitRemote" : "origin", "upStream" : "rtos_sdk/product/aocpu/+/"}, {"blType" : "rtos_scripts", "gitBranch" : "projects/amlogic-dev", "stbBranch" : "projects/openlinux/bl-3.5.0", "gitRemote" : "origin", "upStream" : "rtos_sdk/scripts/+/"}, {"blType" : "rtos_soc", "gitBranch" : "projects/amlogic-dev", "stbBranch" : "projects/openlinux/bl-3.5.0", "gitRemote" : "origin", "upStream" : "rtos_sdk/soc/riscv/+/"}, {"blType" : "bl31_1.3", "gitBranch" : "amlogic-dev-1.3", "stbBranch" : "projects/openlinux/bl-3.5.0", "gitRemote" : "firmware", "upStream" : "ARM-software/arm-trusted-firmware/+/"}, {"blType" : "bl31_2.7", "gitBranch" : "amlogic-dev-2.7", "stbBranch" : "projects/openlinux/bl-3.5.0-2.7", "gitRemote" : "firmware", "upStream" : "ARM-software/arm-trusted-firmware/+/"}, {"blType" : "bl32_3.8", "gitBranch" : "amlogic-dev-3.8.0", "stbBranch" : "projects/openlinux/bl-3.5.0", "gitRemote" : "firmware", "upStream" : "OP-TEE/optee_os/+/"}, {"blType" : "bl32_3.18", "gitBranch" : "amlogic-dev-3.18.0", "stbBranch" : "projects/openlinux/bl-3.5.0-3.18", "gitRemote" : "firmware", "upStream" : "OP-TEE/optee_os/+/"}, {"blType" : "bl33_v2019", "gitBranch" : "amlogic-dev-2019", "stbBranch" : "projects/openlinux/bl-3.5.0", "gitRemote" : "uboot", "upStream" : "uboot/+/"}, {"blType" : "bl33_v2023", "gitBranch" : "amlogic-dev-2023", "stbBranch" : "projects/openlinux/bl-3.5.0-2023", "gitRemote" : "uboot", "upStream" : "uboot/+/"}, {"blType" : "fip", "gitBranch" : "amlogic-dev", "stbBranch" : "projects/openlinux/bl-3.5.0", "gitRemote" : "fip", "upStream" : "amlogic/tools/fip/+/"} ] # the local csv file columns csv_file_column = [ {"ID" : "A", "WIDTH" : 12, "NAME" : "Index"}, {"ID" : "B", "WIDTH" : 45, "NAME" : "Trunk Commit"}, {"ID" : "C", "WIDTH" : 30, "NAME" : "Trunk Cl Link"}, {"ID" : "D", "WIDTH" : 20, "NAME" : "Stb Merged?"}, {"ID" : "D", "WIDTH" : 20, "NAME" : "Is Force Patch?"}, {"ID" : "E", "WIDTH" : 20, "NAME" : "Is Secure Patch?"}, {"ID" : "F", "WIDTH" : 20, "NAME" : "Reviewer"}, {"ID" : "G", "WIDTH" : 16, "NAME" : "Related to other CL ?"}, {"ID" : "H", "WIDTH" : 20, "NAME" : "QA Test Cases"}, {"ID" : "I", "WIDTH" : 16, "NAME" : "QA Verify Result"}, {"ID" : "J", "WIDTH" : 20, "NAME" : "New CL"} ] # git update branch def git_src_update(gitPath, gitRemote, gitBranch): local_path = os.path.join(gitPath) repo = Repo(local_path) try: repo.git.branch('-D', 'test') except: pass try: repo.git.checkout('-b', 'test') except: try: repo.git.checkout('test') except: pass pass try: repo.git.branch('-D', gitBranch) except: pass try: #repo.git.clean('-d', '-fx') repo.git.checkout('-t', 'remotes/' + gitRemote + '/' + gitBranch) except: try: repo.git.checkout(gitBranch) except: exit('Error: check out branch (%s / %s) failed!'%(gitRemote,gitBranch)) pass try: repo.git.branch('-D', 'test') except: pass try: repo.git.fetch('--all') except: pass try: repo.git.reset('--hard', gitRemote + '/' + gitBranch) except: exit('Error: git reset branch (%s / %s) failed!'%(gitRemote,gitBranch)) try: #repo.git.pull(gitRemote, gitBranch) repo.git.pull() except: exit('Error: git pull branch (%s) failed!'%gitBranch) # git update branch def get_bltype_branch_id(bltype, list): for i in range(len(list)): stream_dic = {"blType":None, "gitBranch":None, "gitRemote":None, "upStream":None} stream_dic = list[i] if str(bltype) == str(stream_dic['blType']): print(' > Match the local bltype ID = ', i+1) return i else: return -1 # Prase json array to get commits info def git_commits_to_src_link(): print('\n[TRUNK LIST]:') # prase each repo for i in range(len(trunk_list)): print(' > [%d] gitPath: %-12s lastCommit: %s'%(i+1, trunk_list[i]['gitPath'], trunk_list[i]['lastCommit'])) if len(trunk_list[i]['lastCommit']) == 0: print(' > lastCommit is NULL !') continue try: os.chdir(topdir + trunk_list[i]['gitPath']) except: exit('Error: NO such git path:', trunk_list[i]['gitPath']) # compare trunk list and local blSrcGits, find match id index = get_bltype_branch_id(trunk_list[i]['blType'], blSrcGits) if index < 0: exit("Error: NO found match blType:", trunk_list[i]['blType']) # update stable branch git_src_update(os.getcwd(), blSrcGits[index]['gitRemote'], blSrcGits[index]['stbBranch']) changedId_list = get_stb_changeId(os.getcwd(), 200) # update target branch git_src_update(os.getcwd(), blSrcGits[index]['gitRemote'], blSrcGits[index]['gitBranch']) # run git log format and produce commit list commit_list = git_cmt_parse(os.getcwd(), trunk_list[i]['lastCommit'], 'HEAD', changedId_list, i) git_cmt_2_csv(csvfile, trunk_list[i]['blType'], commit_list, blSrcGits[index], i) # Open json file, prase last commit def prase_json_file(): global trunk_list with open(jsonfile,'r') as load_f: try: json_array = json.load(load_f, object_pairs_hook=OrderedDict) except: exit('Error: Incorrect json format!') trunk_list = [] for item in json_array['source_gits']: try: store_details = {"blType":None, "gitPath":None, "lastCommit":None} store_details['blType'] = item['blType'] store_details['gitPath'] = item['gitPath'] store_details['lastCommit'] = item['lastCommit'] trunk_list.append(store_details) except: exit('Error: get trunk last commit failed.\n') return trunk_list # output instance of str def to_str(bytes_or_str): if isinstance(bytes_or_str, bytes): value = bytes_or_str.decode('utf-8') else: value = bytes_or_str return value # run shell cmd return bytes def bash_command(cmd): process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) #stdout, stderr = process.communicate() return process.stdout.read() # get stable branch changed-id list def get_stb_changeId(gitPath, clNum): local_path = os.path.join(gitPath) repo = Repo(local_path) commit_log = repo.git.log('--pretty={"commit":"%h"}', max_count=clNum) try: log_list = commit_log.split("\n") #if debug_enable: # print(' > %s'%(log_list)) except: pass real_log_list = [eval(str(item)) for item in log_list] changedId_list = [None] * clNum for i in range(len(real_log_list)): try: cmd = 'git log ' + real_log_list[i]['commit'] + ' -1 | grep Change-Id: | head -n 1' res = to_str(bash_command(cmd)).replace('\n', '') if res: changedId_list[i] = res.split("Change-Id:")[1] else: changedId_list[i] = 'NULL' if debug_enable: print(' > [%d] overwrite Change-Id: = %s'%(i,changedId_list[i] )) except: pass #print("changedId_list: ",changedId_list[0]) return changedId_list # get commit list info def git_cmt_parse(gitPath, lastCommit, headCommit, changedId_list, index): local_path = os.path.join(gitPath) repo = Repo(local_path) latest_commit = repo.head.commit.hexsha trunk_list[index]['lastCommit'] = latest_commit # run git log --format commit_log = repo.git.log('--pretty={"summary":"%s","commit":"%h","hash":"%H","author":"%ae","date":"%cd","Change-Id":"null","meged":"null"}', '--reverse', lastCommit + '...' + headCommit) try: log_list = commit_log.split("\n") #if debug_enable: # print(' > %s'%(log_list)) except: pass #Remove invalid commits autosubmit = [] log_list_len = len(log_list) for i in range(log_list_len): if "gerrit.autosubmit@amlogic.com" in log_list[i]: autosubmit.append(i) log_list = [log_list[i] for i in range(log_list_len) if (i not in autosubmit)] # deal with "Merge into" or "revert" commits for i in range(len(log_list)): try: log_list[i] = str(re.sub(r'Merge "', r'Merge <', str(log_list[i]))) log_list[i] = str(re.sub(r'" into', r'> into', str(log_list[i]))) log_list[i] = str(re.sub(r'Revert "', r'Revert <', str(log_list[i]))) log_list[i] = str(re.sub(r'"",', r'>",', str(log_list[i]))) log_list[i] = str(re.sub(r' "', r' "', str(log_list[i]))) log_list[i] = str(re.sub(r'" ', r'" ', str(log_list[i]))) log_list[i] = str(re.sub(r' `', r' ', str(log_list[i]))) log_list[i] = str(re.sub(r'&', r' ', str(log_list[i]))) log_list[i] = str(re.sub(r'loc\'', r'loc', str(log_list[i]))) log_list[i] = str(re.sub(r'">"', r'>"', str(log_list[i]))) log_list[i] = str(re.sub(r'">', r' >', str(log_list[i]))) log_list[i] = str(re.sub(r'("colorattribute")', r"'colorattribute'", str(log_list[i]))) except: pass #print(' > [%d] %s'%(i,log_list[i])) # eval special special characters real_log_list=[] for index, item in enumerate(log_list): try: real_log_list.append(eval(str(item))) except(ValueError, SyntaxError) as e: real_log_list = [] print(' > eval(str(item)) ERROR!') print(' > %s'%(item)) #print(f" > Error details: {e}") pass # update real_log_list[i]['Change-Id'] with JiraNo for j in range(len(real_log_list)): try: cmd = 'git log ' + real_log_list[j]['commit'] + ' -1 | grep Change-Id: | head -n 1' res = to_str(bash_command(cmd)).replace('\n', '') if res: real_log_list[j]['Change-Id'] = res.split("Change-Id:")[1] if real_log_list[j]['Change-Id'] in changedId_list : real_log_list[j]['meged'] = '1' else: real_log_list[j]['Change-Id'] = 'NULL' if debug_enable: print(' > [%d] overwrite Change-Id: = %s'%(j,real_log_list[j]['Change-Id'])) except: print("Change-Id get bypass") pass try: print(' > Commit list max number of rows = ', len(real_log_list)) except: pass return real_log_list # save commit info to csv def git_cmt_2_csv(csvfile, blType, commit_list, stream_dic, sheet_index): global alignment # Load a workbook object wb = openpyxl.load_workbook(csvfile) # Creat csv sheet named by blType title = re.sub(r'[?|$|.|!|/|*]', r'_', blType) sheet = wb.create_sheet(title, sheet_index) sheet.title = title # Set active sheet wb.active = sheet_index try: # Set aligment alignment = Alignment(horizontal="left", vertical="center", wrap_text=True) for i in range(len(commit_list) + 1): for j in range(len(csv_file_column)): sheet.cell(row = i + 1, column = j + 1).alignment = alignment except: pass # set cell(1,1): value and font font = Font(size=11, bold=True) sheet.cell(row = 1, column = 1).value = blType + " " + csv_file_column[0]['NAME'] sheet.cell(row = 1, column = 1).font = font # set row 1: value and font for i in range(1, len(csv_file_column)): sheet.cell(row = 1, column = i + 1).value = csv_file_column[i]['NAME'] sheet.cell(row = 1, column = i + 1).font = font # set row 1: height # sheet.row_dimensions[1].height = 30 # set row 2-n: trunk commit for i in range(len(commit_list)): try: # column 1: ID index sheet.cell(row = i + 2, column = 1).value = i + 1 sheet.cell(row = i + 2, column = 2).value = \ commit_list[i]['summary'] + '\n'\ 'Commit: ' + commit_list[i]['commit'] + '\n'\ 'Change-Id: ' + commit_list[i]['Change-Id'] + '\n'\ 'Author: ' + commit_list[i]['author'] + '\n'\ 'Date: ' + commit_list[i]['date'] # column 3：Trunk Cl Link sheet.cell(row = i + 2, column = 3).value = scgitPrefix + \ stream_dic['upStream'] + \ commit_list[i]['hash'] sheet.cell(row = i + 2, column = 4).value = commit_list[i]['meged'] except: pass # set column A-I width for i in range(len(csv_file_column)): sheet.column_dimensions[csv_file_column[i]['ID']].width = csv_file_column[i]['WIDTH'] # save the file wb.save(csvfile) # Creat csv file to save cmt list def create_csv_file(outdir, inputfile): global csvfile # get csv file full name localTime = time.strftime("_%Y%m%d_%H%M%S", time.localtime()) base_name=os.path.basename(inputfile) file_name = os.path.splitext(base_name)[0] csvfile = str(topdir) + str(outdir) + "/" + file_name + str(localTime) + ".xlsx" # creat csv file, overwrite existing files wb = openpyxl.Workbook() wb.save(csvfile) # get argv def getOptions(args=sys.argv[1:]): global jsonfile global debug_enable # crate prase progress parser = argparse.ArgumentParser(description="Get all blx commits and save to csv file.") # add argv support parser.add_argument("-j", "--jsoncfg", required=True, help="Your input json file.") parser.add_argument("-o", "--outDir", required=True, help="Your output .xlsx dir.") parser.add_argument("-v", "--verbose", required=False, help="Increase output verbosity.", action="store_true") parser.add_argument("-p", "--push", required=False, help="Push csv file to confluence.", action="store_true") # prase argv options = parser.parse_args(args) # check debug on if options.verbose: debug_enable = 1 else: debug_enable = 0 # check whether the outdir is available jsonfile = str(options.jsoncfg) if not os.path.isdir(options.outDir): exit('Error: No such directory!') # creat new csv file create_csv_file(options.outDir, options.jsoncfg) print('\n[CONFIG INFO]:') print(' > Run in toplevel : ', topdir) print(' > Input json file : ', options.jsoncfg) print(' > Output xlsx dir : ', options.outDir) print(' > Output xlsx name: ', csvfile) # Get the top-level directory which include .repo def get_top_dir(): global topdir pwd = os.getcwd() dirName = 'fip' if not os.path.exists(dirName): topdir = pwd + "/../" else: topdir = pwd + "/" return topdir # Record summary info, and save to scv file def record_in_summary_sheet(): wb = openpyxl.load_workbook(csvfile) default_ws = 'Sheet' # active for s in range(len(wb.sheetnames)): if wb.sheetnames[s] == default_ws: break try: wb.active = s sheet = wb[default_ws] sheet.title = 'summary' except: exit('Error: NOT found sheet: %s'%default_ws) # set column width try: sheet.column_dimensions['A'].width = 10 sheet.column_dimensions['B'].width = 20 sheet.column_dimensions['C'].width = 15 except: pass # row 1: summary head sheet.cell(row = 1, column = 1).value = 'ID' sheet.cell(row = 1, column = 2).value = 'title' sheet.cell(row = 1, column = 3).value = 'max_row' # row n: summary info for j in range(0,len(summary_list)): try: sheet.cell(row = j+2, column = 1).value = summary_list[j]['ID'] sheet.cell(row = j+2, column = 2).value = summary_list[j]['title'] sheet.cell(row = j+2, column = 3).value = summary_list[j]['max_row'] if debug_enable: print('[%d] id:%d, title:%s, title:%2d'%(j+1, summary_list[j]['ID'], summary_list[j]['title'], summary_list[j]['max_row'])) except: pass # Set aligment for i in range(0, len(summary_list) + 1): for j in range(0, 3): try: sheet.cell(row = i + 1, column = j + 1).alignment = alignment except: pass wb.save(csvfile) return # Printf scv file , such as max list num def summary_for_scv_sheets(): global summary_list wb = openpyxl.load_workbook(csvfile) summary_list = [] print('\n[SUMMARY SHEETS]:') print('=========================================') for i in range(0, len(wb.sheetnames)-1): try: sheet = wb[wb.sheetnames[i]] print('[%d] Sheet: %-12s Max_Row: %2d'%(i+1, sheet.title, sheet.max_row-1)) summary_details = {"ID":0, "title":None, "max_row":0} summary_details['ID'] = i+1 summary_details['title'] = sheet.title summary_details['max_row'] = sheet.max_row-1 summary_list.append(summary_details) except: pass print('=========================================\n') #print('summary_list: ', summary_list) wb.save(csvfile) return def update_stable_json(): latest_list={ "source_gits":trunk_list } try: with open(jsonfile, 'w') as json_file: json.dump(latest_list, json_file, ensure_ascii=False, indent=4) except Exception as e: print("write json failed!!: ", e) # Main func if __name__ == "__main__": # Set stderr with color from IPython.core.ultratb import ColorTB sys.excepthook = ColorTB() # get repo top dir get_top_dir() # Prase argv in / out getOptions(sys.argv[1:]) # Prase json file to get last commit prase_json_file() # Get commits list and src link git_commits_to_src_link() # Output csv summary summary_for_scv_sheets() # Record in summary sheet record_in_summary_sheet() update_stable_json() print('OUTPUT csv: ', os.path.basename(csvfile)) exit('RUN OK !')