#!/usr/bin/env python # -*- coding: utf-8 -*- # Tarang dependency installer # # Copyright (C) 2008, 2009 Mahendra K. Verma # # Mahendra K. Verma # Indian Institute of Technology, Kanpur-208016 # UP, India # # mkv@iitk.ac.in # # This file is part of Tarang-2 . # # Tarang-2 is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # Tarang-2 is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Tarang-2; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, U # # \file tarang_dependency_installer.sh # \author Anando G. Chatterjee, Rishabh Sahu # \date 26 July 2018 from __future__ import division from __future__ import print_function import os import re import sys import time import signal import urllib import hashlib import tarfile import threading import subprocess ################################ # For usage type # $ ./tarang_dependency_installer.py --help ################################ ############### # Determine system specific properties and configurations # This class helps it to work across various platforms and python versions ############### class System: EXIT_CODES={} CWD = "." # path of this script def __init__(self): self.EXIT_CODES={ "SUCCESS": 0, "SIGINT": 1, # Ctrl+C is pressed "SIGTERM": 2, # Terminate signal is send "TIMEOUT": 3, # Download timeouts "DOWNLOAD_FAIL": 4, "MD5_SUM_FAIL": 5, "EXTRACTION_FAIL": 6, "COMPILER_NOT_FOUND": 7, "INSTALLATION_FAIL": 8 } self.CWD = os.path.dirname(os.path.realpath(__file__)) # Register the signal handlers signal.signal(signal.SIGTERM, self.shutdown) signal.signal(signal.SIGINT, self.shutdown) def exit(self, process): sys.exit(self.EXIT_CODES[process]) def command_exists(self, command, path): # Python 3.3+ try: import shutil return shutil.which(command, path=path) is not None except (ImportError, NotImplementedError, AttributeError): pass # Older python for path in path.split(os.pathsep): if os.access(os.path.join(path, command), os.X_OK): return True return False def remove(self, file): try: os.remove(os.path.join(self.CWD, file)) except OSError: pass def shutdown(self, signum, frame): if signum == signal.SIGINT: self.exit('SIGINT') else: self.exit('SIGTERM') #################### # Test if terminal supports ANSI colours # https://stackoverflow.com/a/22254892 #################### def check_colours_available(self): stdout_is_a_tty = False if hasattr(sys.stdout, 'isatty') and sys.stdout.isatty(): stdout_is_a_tty = sys.stdout.isatty() # If output is not a terminal disable self.colours if stdout_is_a_tty and Parameters.USE_COLOR.lower() == "yes": return True else: return False ################ # https://stackoverflow.com/a/1006301 ################ def available_cpu_count(self): """ Number of available virtual or physical CPUs on this system""" # cpuset # cpuset may restrict the number of *available* processors try: m = re.search(r'(?m)^Cpus_allowed:\s*(.*)$', open('/proc/self/status').read()) if m: res = bin(int(m.group(1).replace(',', ''), 16)).count('1') if res > 0: return res except IOError: pass # Python 2.6+ try: import multiprocessing return multiprocessing.cpu_count() except (ImportError, NotImplementedError): pass # POSIX try: res = int(os.sysconf('SC_NPROCESSORS_ONLN')) if res > 0: return res except (AttributeError, ValueError): pass # BSD try: sysctl = subprocess.Popen(['sysctl', '-n', 'hw.ncpu'], stdout=subprocess.PIPE) scStdout = sysctl.communicate()[0] res = int(scStdout) if res > 0: return res except (OSError, ValueError): pass # Linux try: res = open('/proc/cpuinfo').read().count('processor\t:') if res > 0: return res except IOError: pass # Solaris try: pseudoDevices = os.listdir('/devices/pseudo/') res = 0 for pd in pseudoDevices: if re.match(r'^cpuid@[0-9]+$', pd): res += 1 if res > 0: return res except OSError: pass # Other UNIXes (heuristic) try: try: dmesg = open('/var/run/dmesg.boot').read() except IOError: dmesgProcess = subprocess.Popen(['dmesg'], stdout=subprocess.PIPE) dmesg = dmesgProcess.communicate()[0] res = 0 while '\ncpu' + str(res) + ':' in dmesg: res += 1 if res > 0: return res except OSError: pass return None system = System() class Parameters: CC = os.getenv('CC' , 'gcc') CXX = os.getenv('CXX' , 'g++') FC = os.getenv('FC' , 'gfortran') MPICC = os.getenv('MPICC' , 'mpicc') MPICXX = os.getenv('MPICXX', 'mpicxx') MPIFC = os.getenv('MPIFC' , 'mpif90') PREFIX = os.getenv('PREFIX' , os.getenv('HOME') + '/local') OPTIONS = os.getenv('OPTIONS' , '9') UPDATE_RC_FILE = os.getenv('UPDATE_RC_FILE' , 'yes') # yes/no J = os.getenv('J' , str(system.available_cpu_count())) # Number of process to be spawned during make LOG = os.getenv('LOG' , 'yes') # yes/no LOG_FILE = os.getenv('LOG_FILE' , os.path.splitext(os.path.basename(sys.argv[0]))[0] + ".log") RC_FILE = os.getenv('RC_FILE', os.getenv('HOME') + '/.bashrc') TIMEOUT = os.getenv('TIMEOUT', '30') # It will wait 'n' seconds for download server to reply USE_UTF8 = os.getenv('USE_UTF8', 'yes') # Use UTF-8 symbols for Progress DONE and FAIL USE_COLOR = os.getenv('USE_COLOR', 'yes') SELECT_DEFAULT = False # Assume default values for all questions # https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_(Select_Graphic_Rendition)_parameters # https://en.wikipedia.org/wiki/ANSI_escape_code#3/4_bit # \033 is Octal code for ESC class Colours: colours_available = True def __init__(self): if system.check_colours_available(): self.colours_available = True else: self.colours_available = False self.update() def enable(self): self.CLEAR = "\033[0m" # clear text [ reset all formatting self.MAIN = "\033[1;34m" # main text [ bold; blue self.OTHER = "\033[;34m" # prefix & brackets [ blue self.SEPARATOR = "\033[1;30m" # separator [ bold; black self.SECTION = self.CLEAR # section [ same as CLEAR self.PROGRESS = "\033[;1m" # percent [ bold self.BUSY = "\033[;36m" # busy [ cyan self.FAIL = "\033[;31m" # failed [ red self.DONE = "\033[;32m" # completed [ green self.BKGD = "\033[1;35m" # backgrounded [ bold; magenta self.H1 = self.MAIN # highlight text 1 [ same as MAIN self.H2 = "\033[1;36m" # highlight text 2 [ bold; cyan def disable(self): self.CLEAR = "" # clear text self.MAIN = "" # main text self.OTHER = "" # prefix & brackets self.SEPARATOR = "" # separator self.SECTION = "" # section self.PROGRESS = "" # percent self.BUSY = "" # busy self.FAIL = "" # failed self.DONE = "" # completed self.BKGD = "" # backgrounded self.H1 = "" # highlight text 1 self.H2 = "" # highlight text 2 def update(self): if self.colours_available: self.enable() else: self.disable() # This function prints to screen and writes it to 'log file' # and any other file added through 'add_file' # Its functions can be invoked without instantiating it '@classmethod' class Logger: file_logging = False # Extract script name without extension # https://stackoverflow.com/a/1228618 log_file_name = os.path.splitext(os.path.basename(sys.argv[0]))[0] + ".log" other_files = [] colours = Colours() @classmethod def set_log_file_name(cla, log_file_name): cla.log_file_name = log_file_name @classmethod def disable_file_logging(cla): cla.file_logging = False @classmethod def enable_file_logging(cla): cla.file_logging = True @classmethod def add_file(cla, file_name): cla.other_files.append(file_name) @classmethod def remove_file(cla, file_name): if file_name in cla.other_files: cla.other_files.remove(file_name) @classmethod def write_to_file(cla, text): if cla.file_logging: try: file = open(cla.log_file_name, "a") file.write(text) file.flush() file.close() except IOError: self.write_to_screen("Logger - IOError: Unable to write to '" + cla.log_file_name + "'") sys.exit(1) if cla.other_files: for file_name in cla.other_files: try: file = open(file_name, "a") file.write(text) file.flush() file.close() except IOError: self.write_to_screen("Logger - IOError: Unable to write to '" + file_name + "'") sys.exit(1) @classmethod def write_to_screen(cla, text): print(text, end='') sys.stdout.flush() @classmethod def log_parameters(cla): cla.write_to_file('\n') cla.write_to_file('--------------------------------\n') cla.write_to_file('Initial configuration:\n') cla.write_to_file('--------------------------------\n') cla.write_to_file(''' CC="%s" CXX="%s" FC="%s" MPICC="%s" MPICXX="%s" MPIFC="%s" PREFIX="%s" OPTIONS="%s" UPDATE_RC_FILE="%s" J="%s" LOG="%s" LOG_FILE="%s" RC_FILE="%s" TIMEOUT="%s" USE_UTF8="%s" USE_COLOR="%s" SELECT_DEFAULT="%r" ''' % ( Parameters.CC, Parameters.CXX, Parameters.FC, Parameters.MPICC, Parameters.MPICXX, Parameters.MPIFC, Parameters.PREFIX, Parameters.OPTIONS, Parameters.UPDATE_RC_FILE, Parameters.J, Parameters.LOG, Parameters.LOG_FILE, Parameters.RC_FILE, Parameters.TIMEOUT, Parameters.USE_UTF8, Parameters.USE_COLOR, Parameters.SELECT_DEFAULT )) cla.write_to_file('--------------------------------\n') cla.write_to_file('\n') @classmethod def head(cla, text, end='\n'): cla.write_to_file('\n' + str(text) + end) text = "".join([cla.colours.H1, str(text), cla.colours.CLEAR]) print('\n' + text + end, end='') sys.stdout.flush() @classmethod def subhead(cla, text, end='\n'): cla.write_to_file(str(text) + end) text = "".join([cla.colours.H2, str(text), cla.colours.CLEAR]) print(text + end, end='') sys.stdout.flush() @classmethod def section(cla, text, end='\n'): cla.write_to_file(str(text) + end) text = "".join([cla.colours.SECTION, str(text), cla.colours.CLEAR]) print(text + end, end='') sys.stdout.flush() @classmethod def log(cla, text, end='\n'): cla.write_to_file(str(text) + end) cla.write_to_screen(str(text) + end) @classmethod def error(cla, text, end='\n'): cla.write_to_file(str(text) + end) fail_text = "".join([cla.colours.FAIL, str(text), cla.colours.CLEAR]) sys.stderr.write(fail_text + end) sys.stderr.flush() @classmethod def is_installed(cla, library): try: log = open(cla.log_file_name).read() except IOError: return False if log.find(library + ' Installation finished') >=0: return True else: return False @classmethod def start_installing(cla, library): cla.head(library) Logger.write_to_file("\n") @classmethod def finish_installing(cla, library): Logger.write_to_file("\n"+ library +" Installation finished\n\n") # This class shows what percent of the current operation has completed # This function can also show a loop when download waits for the server to respond class Progress: loop = None previous_term_handler = signal.SIG_DFL previous_int_handler = signal.SIG_DFL def __init__(self, title="", prefix=""): self.title = title self.prefix = prefix self.colours = Colours() self.progress_elements = [] # Register the signal handlers signal.signal(signal.SIGUSR1, self.shutdown_loop) self.previous_term_handler = signal.signal(signal.SIGTERM, self.shutdown_loop) self.previous_int_handler = signal.signal(signal.SIGINT, self.shutdown_loop) if Parameters.USE_UTF8.lower() == 'yes': self.DONE = " \xe2\x9c\x94 " # utf-8 encoded symbol: U+2714 self.FAIL = " \xe2\x9c\x98 " # utf-8 encoded symbol: U+2718 else: self.DONE = "DONE" self.FAIL = "FAIL" def __exit__(self): signal.signal(signal.SIGTERM, self.previous_term_handler) signal.signal(signal.SIGINT, self.previous_int_handler) class Loop(threading.Thread): def __init__(self, title="", prefix=""): threading.Thread.__init__(self) # The shutdown_flag is a threading.Event object that # indicates whether the thread should be terminated. self.shutdown_flag = threading.Event() self.title = title self.prefix = prefix self.colours = Colours() def update_title(self, title): self.title = title def update_progress_elements(self, progress_format, title_colour): self.progress_elements = progress_format + [self.colours.OTHER, self.prefix, title_colour, "%s", self.colours.CLEAR] def text(self, text): self.colours.update() self.update_progress_elements([self.colours.OTHER, "[", self.colours.BUSY, "%s", self.colours.OTHER, "]", self.colours.CLEAR], self.colours.PROGRESS) Logger.write_to_screen("\r" + "".join(self.progress_elements) % (text, self.title)) def run(self): stages = ['* ', ' * ', ' * ', ' *', ' * ', ' * ',] i = 0 num_stages = len(stages) seconds_elapsed = 0 while True: if self.shutdown_flag.is_set(): break self.text(stages[i]) time.sleep(0.5) seconds_elapsed += 0.5 # send signal to master process on TIMEOUT if int(seconds_elapsed) >= int(Parameters.TIMEOUT): os.kill(os.getpid(), signal.SIGUSR1) i += 1 i = i % num_stages def update_title(self, title): self.title = title def update_progress_elements(self, progress_format, title_colour): self.progress_elements = progress_format + [self.colours.OTHER, self.prefix, title_colour, "%s", self.colours.CLEAR] def start_loop(self): self.stop_loop() self.loop = self.Loop(self.title, self.prefix) self.loop.start() def stop_loop(self): if self.loop: self.loop.shutdown_flag.set() self.loop.join() def shutdown_loop(self, signum, frame): self.stop_loop() print('\n', end='') sys.stdout.flush() if signum == signal.SIGUSR1: Logger.error('Connection timeout') system.exit('TIMEOUT') # restore previous handlers and send the signal again self.__exit__() os.kill(os.getpid(), signum) def percent(self, n): self.colours.update() progress_format = [self.colours.OTHER, "[", self.colours.PROGRESS, "%3d%%", self.colours.OTHER, "]", self.colours.CLEAR] self.update_progress_elements(progress_format, self.colours.PROGRESS) Logger.write_to_screen("\r" + "".join(self.progress_elements) % (n, self.title)) def text(self, text): self.colours.update() progress_format = [self.colours.OTHER, "[", self.colours.PROGRESS, "%s", self.colours.OTHER, "]", self.colours.CLEAR] self.update_progress_elements(progress_format, self.colours.PROGRESS) Logger.write_to_screen("\r" + "".join(self.progress_elements) % (text, self.title)) def finish(self, text=" "): self.colours.update() progress_format = [self.colours.OTHER, "[", self.colours.DONE, "%s", self.colours.OTHER, "]", self.colours.CLEAR] self.update_progress_elements(progress_format, self.colours.SECTION) Logger.write_to_screen("\r" + "".join(self.progress_elements) % (self.DONE, self.title) + "\n") self.colours.disable() progress_format = [self.colours.OTHER, "[", self.colours.DONE, "%s", self.colours.OTHER, "]", self.colours.CLEAR] self.update_progress_elements(progress_format, self.colours.SECTION) Logger.write_to_file("".join(self.progress_elements) % (self.DONE, self.title) + "\n") def fail(self, text=" "): self.colours.update() progress_format = [self.colours.OTHER, "[", self.colours.FAIL, "%s", self.colours.OTHER, "]", self.colours.CLEAR] self.update_progress_elements(progress_format, self.colours.SECTION) Logger.write_to_screen("\r" + "".join(self.progress_elements) % (self.FAIL, self.title) + "\n") self.colours.disable() progress_format = [self.colours.OTHER, "[", self.colours.FAIL, "%s", self.colours.OTHER, "]", self.colours.CLEAR] self.update_progress_elements(progress_format, self.colours.SECTION) Logger.write_to_file("".join(self.progress_elements) % (self.FAIL, self.title) + "\n") class Utilities: @classmethod def help(cla): Logger.log('') Logger.log(sys.argv[0] + ' [-h,--help] [-d,--default]') Logger.log('') Logger.log('-h, --help: Display this message.') Logger.log('') Logger.log('-d, --default: Select default values for all questions.') Logger.log('The following environment variables may be used to alter default values.') Logger.log('Default values are given in "double quotes".') Logger.log('') Logger.log('CC="' + Parameters.CC + '"') Logger.log('CXX="' + Parameters.CXX + '"') Logger.log('FC="' + Parameters.FC + '"') Logger.log('MPICC="' + Parameters.MPICC + '"') Logger.log('MPICXX="' + Parameters.MPICXX + '"') Logger.log('MPIFC="' + Parameters.MPIFC + '"') Logger.log('') Logger.log('PREFIX="' + Parameters.PREFIX + '"') Logger.log('OPTIONS="' + Parameters.OPTIONS + '"') Logger.log('UPDATE_RC_FILE="' + Parameters.UPDATE_RC_FILE + '" (yes/no)') Logger.log('J="' + Parameters.J + '" - Number of process to be used during make.') Logger.log('') Logger.log('LOG="' + Parameters.LOG + '" (yes/no)') Logger.log('LOG_FILE="' + Parameters.LOG_FILE + '"') Logger.log('RC_FILE="' + Parameters.RC_FILE + '"') Logger.log('') Logger.log('TIMEOUT="' + Parameters.TIMEOUT + '" - Wait "n" seconds for download server to reply.') Logger.log('USE_UTF8="' + Parameters.USE_UTF8 + '" (yes/no) - Use utf-8 sybmols for showing progress') Logger.log('USE_COLOR="' + Parameters.USE_COLOR + '" (yes/no)') Logger.log('') Logger.log('Note: This script checks in log file to see which libraries installed sucessfully and which of them failed in previous run.') Logger.log('') ###### # Ask a question and log it # If not answered, return the default value ###### @classmethod def ask(cla, question, default_value): answer = raw_input(question) Logger.write_to_file(question + answer + "\n") if not answer: answer = default_value return answer ############## # Download a file and show progress # https://blog.shichao.io/2012/10/04/progress_speed_indicator_for_urlretrieve_in_python.html ############## @classmethod def download(cla, message, url, filename): progress = Progress(message + " (connecting to server)") progress.start_loop() def urlretrieve_reporthook(count, block_size, total_size): global start_time global progress_size global speed global duration if count == 0: progress.stop_loop() progress.percent(0) start_time = time.time() return duration = time.time() - start_time progress_size = int(count * block_size) speed = int(progress_size / (1024 * duration)) percent = int(count * block_size * 100 / total_size) percent = min(percent,100) progress.update_title(message + " (%d MB, %d KB/s, %d seconds elapsed) " % (progress_size/(1024*1024), speed, duration)) progress.percent(percent) try: filename, headers = urllib.urlretrieve(url, filename, urlretrieve_reporthook) except IOError: progress.stop_loop() progress.fail("FAIL") Logger.error('Unable to connect with download server. Network problem.') system.exit("DOWNLOAD_FAIL") return progress.stop_loop() #The login page is downloaded in case the computer is behind a proxy server. #The type of this page is typically 'text/html' if headers.getmaintype() == 'text': progress.fail("FAIL") Logger.error('Unable to connect with download server. You might need to login to transparent proxy.') system.remove(filename) system.exit("DOWNLOAD_FAIL") progress.update_title(message + " (%d MB, %d KB/s, %d seconds elapsed) " % (progress_size/(1024*1024), speed, duration)) progress.finish("DONE") ############## ############# # https://stackoverflow.com/a/3431838 ############# @classmethod def validate_md5(cla, message, file_name, check_hash): progress = Progress(message) progress.percent(0) total_size = os.path.getsize(file_name) bytes_read = 0 try: hash_md5 = hashlib.md5() with open(file_name, "rb") as f: for chunk in iter(lambda: f.read(4096), b""): hash_md5.update(chunk) bytes_read += 4096 percent = int(bytes_read/total_size*100) percent = min(percent, 100) progress.percent(percent) except IOError: progress.fail("FAIL") system.exit("MD5_SUM_FAIL") return if not (hash_md5.hexdigest() == check_hash): progress.fail("FAIL") Logger.error("Validation of MD5 sum faild.") system.exit("MD5_SUM_FAIL") return progress.finish("DONE") ############## ############# # https://stackoverflow.com/a/31163747 ############# @classmethod def untar(cla, message, file_name): progress = Progress(message) progress.percent(0) try: tar = tarfile.open(file_name) num_files = len(tar.getmembers()) file_num = 0 for member in tar.getmembers(): file_num += 1 tar.extract(member) if file_num % 10 == 0: progress.percent(min(int(file_num/num_files*100),100)) tar.close() except tarfile.TarError: progress.fail("FAIL") system.exit("EXTRACTION_FAIL") return progress.finish("DONE") ############## class Installers: env = os.environ.copy() def __init__(self): self.env['PKG_CONFIG_DISABLE_UNINSTALLED'] = 'true' self.env['PATH'] = Parameters.PREFIX + '/bin:' + self.env.get('PATH', '') self.env['PKG_CONFIG_PATH'] = Parameters.PREFIX + '/lib/pkgconfig:' + self.env.get('PKG_CONFIG_PATH', '') self.env['HDF5_ROOT'] = Parameters.PREFIX + ':' + self.env.get('HDF5_ROOT', '') self.env['CPATH'] = Parameters.PREFIX + '/include:' + self.env.get('CPATH', '') self.env['LD_LIBRARY_PATH'] = Parameters.PREFIX + '/lib:' + self.env.get('LD_LIBRARY_PATH', '') self.env['LIBRARY_PATH'] = Parameters.PREFIX + '/lib:' + self.env.get('LIBRARY_PATH', '') self.env['MANPATH'] = Parameters.PREFIX + '/share/man:' + self.env.get('MANPATH', '') def Show_progress(self, step_name, installer_path, executable, out_file_name, err_file_name, step_env, max_lines, step_size=10): progress = Progress(step_name, " :: ") progress.percent(0) try: out_file = open(installer_path + '/' + out_file_name, 'w') err_file = open(installer_path + '/' + err_file_name, 'w') except IOError: progress.fail("FAIL") system.exit("INSTALLATION_FAIL") return try: process = subprocess.Popen(executable, stdout=subprocess.PIPE, stderr=err_file, env=step_env, cwd=installer_path) except OSError: progress.fail("FAIL") out_file.close() err_file.close() Logger.error('Unable to execute "' + " ".join(executable) + '"') Logger.error('In directory "' + os.path.join(system.CWD, installer_path) + '"') system.exit("INSTALLATION_FAIL") return # https://stackoverflow.com/a/2813530 line_count=0 while True: line = process.stdout.readline() if line != '': line_count += 1 out_file.write(line.decode('utf-8')) out_file.flush() if line_count % step_size == 0: percent = int((line_count/max_lines)*100) percent = min(percent, 100) progress.percent(percent) else: break out_file.close() err_file.close() if process.wait() == 0: progress.finish("DONE") else: progress.fail("FAIL") Logger.error(step_name + ' failed. Please see the following files for detail.') Logger.error(os.path.join(system.CWD, installer_path, out_file_name)) Logger.error(os.path.join(system.CWD, installer_path, err_file_name)) system.exit("INSTALLATION_FAIL") def Install_library_with_configure(self, installer_path, library_env, max_lines, config_options=None, message=None): if not message: Logger.section(" Installing") else: Logger.section(message) configure = ['./configure','--prefix=' + Parameters.PREFIX] if config_options: configure += config_options self.Show_progress("configure", installer_path, configure , 'config.out', 'config.err', step_env=library_env, max_lines=max_lines[0]) self.Show_progress("make", installer_path, ['make', '-j', Parameters.J], 'make.out', 'make.err', step_env=library_env, max_lines=max_lines[1]) self.Show_progress("make install", installer_path, ['make', 'install'], 'make_install.out', 'make_install.err', step_env=library_env, max_lines=max_lines[2]) def Install_library_with_cmake(self, installer_path, library_env, max_lines, config_options=None, message=None, file_suffix=''): if not message: Logger.section(" Installing") else: Logger.section(message) cmake = ['cmake','-DCMAKE_INSTALL_PREFIX=' + Parameters.PREFIX] if config_options: cmake += config_options system.remove(os.path.join(installer_path, 'CMakeCache.txt')) self.Show_progress("cmake", installer_path, cmake , 'cmake.out', 'cmake.err', step_env=library_env, max_lines=max_lines[0]) self.Show_progress("make", installer_path, ['make', '-j', Parameters.J], 'make.out', 'make.err', step_env=library_env, max_lines=max_lines[1]) self.Show_progress("make install", installer_path, ['make', 'install'], 'make_install.out', 'make_install.err', step_env=library_env, max_lines=max_lines[2]) def Install_cmake(self): CMAKE_VERSION="3.0.2" CMAKE_TAR_FILE="cmake-2.8.12.tar.gz" CMAKE_URL="http://turbulencehub.org/wp-content/uploads/Download_Files/cmake-2.8.12.tar.gz" CMAKE_INSTALLER_PATH="cmake-2.8.12" CMAKE_MD5SUM="105bc6d21cc2e9b6aff901e43c53afea" if (Logger.is_installed("CMAKE - " + CMAKE_VERSION)): Logger.head("Skipping CMAKE") return Logger.start_installing("CMAKE - " + CMAKE_VERSION) if not os.path.isfile(CMAKE_TAR_FILE): Utilities.download(" Downloading", CMAKE_URL, CMAKE_TAR_FILE) Utilities.validate_md5(" Validating MD5 sum", CMAKE_TAR_FILE, CMAKE_MD5SUM) Utilities.untar(" Extracting", CMAKE_TAR_FILE) if os.path.isdir(CMAKE_INSTALLER_PATH): env = self.env.copy() env['CC'] = Parameters.CC env['CXX'] = Parameters.CXX if not system.command_exists(env['CC'], env['PATH']): Logger.error('CC="' + env['CC'] + '" not found.') system.exit('COMPILER_NOT_FOUND') if not system.command_exists(env['CXX'], env['PATH']): Logger.error('CXX="' + env['CXX'] + '" not found.') system.exit('COMPILER_NOT_FOUND') self.Install_library_with_configure(CMAKE_INSTALLER_PATH, library_env=env, max_lines=[1198, 500, 1680]) Logger.finish_installing("CMAKE - " + CMAKE_VERSION) else: Logger.head("CMAKE installer not found.") def Install_blitz(self): BLITZ_VERSION="1.0.1" BLITZ_TAR_FILE="blitz-1.0.1.tar.gz" BLITZ_URL="http://turbulencehub.org/wp-content/uploads/Download_Files/blitz-1.0.1.tar.gz" BLITZ_INSTALLER_PATH="blitz-1.0.1" BLITZ_MD5SUM="fe43e2cf6c9258bc8b369264dd008971" if (Logger.is_installed("Blitz++ - " + BLITZ_VERSION)): Logger.head("Skipping Blitz++") return Logger.start_installing("Blitz++ - " + BLITZ_VERSION) if not os.path.isfile(BLITZ_TAR_FILE): Utilities.download(" Downloading", BLITZ_URL, BLITZ_TAR_FILE) Utilities.validate_md5(" Validating MD5 sum", BLITZ_TAR_FILE, BLITZ_MD5SUM) Utilities.untar(" Extracting", BLITZ_TAR_FILE) if os.path.isdir(BLITZ_INSTALLER_PATH): env = self.env.copy() env['CC'] = Parameters.CC env['CXX'] = Parameters.CXX if not system.command_exists(env['CC'], env['PATH']): Logger.error('CC="' + env['CC'] + '" not found.') system.exit('COMPILER_NOT_FOUND') if not system.command_exists(env['CXX'], env['PATH']): Logger.error('CXX="' + env['CXX'] + '" not found.') system.exit('COMPILER_NOT_FOUND') self.Install_library_with_configure(BLITZ_INSTALLER_PATH, library_env=env, max_lines=[316, 93, 318]) Logger.finish_installing("Blitz++ - " + BLITZ_VERSION) else: Logger.head("Blitz++ installer not found.") def Install_yaml(self): YAML_VERSION="0.3.0" YAML_TAR_FILE="yaml-cpp-release-0.3.0.tar.gz" YAML_URL="http://turbulencehub.org/wp-content/uploads/Download_Files/yaml-cpp-release-0.3.0.tar.gz" YAML_INSTALLER_PATH="yaml-cpp-release-0.3.0" YAML_MD5SUM="0c0496b195299e956056430444e237b9" if (Logger.is_installed("YAML - " + YAML_VERSION)): Logger.head("Skipping YAML") return Logger.start_installing("YAML-CPP - " + YAML_VERSION) if not os.path.isfile(YAML_TAR_FILE): Utilities.download(" Downloading", YAML_URL, YAML_TAR_FILE) Utilities.validate_md5(" Validating MD5 sum", YAML_TAR_FILE, YAML_MD5SUM) Utilities.untar(" Extracting", YAML_TAR_FILE) if os.path.isdir(YAML_INSTALLER_PATH): env = self.env.copy() env['CC'] = Parameters.CC env['CXX'] = Parameters.CXX if not system.command_exists(env['CC'], env['PATH']): Logger.error('CC="' + env['CC'] + '" not found.') system.exit('COMPILER_NOT_FOUND') if not system.command_exists(env['CXX'], env['PATH']): Logger.error('CXX="' + env['CXX'] + '" not found.') system.exit('COMPILER_NOT_FOUND') self.Install_library_with_cmake(YAML_INSTALLER_PATH, library_env=env, max_lines=[15, 43, 36]) Logger.finish_installing("YAML-CPP - " + YAML_VERSION) else: Logger.head("YAML-CPP installer not found.") def Install_mpi(self): MPI_VERSION="3.1.3" MPI_TAR_FILE="mpich-3.1.3.tar.gz" MPI_URL="http://turbulencehub.org/wp-content/uploads/Download_Files/mpich-3.1.3.tar.gz" MPI_INSTALLER_PATH="mpich-3.1.3" MPI_MD5SUM="93cb17f91ac758cbf9174ecb03563778" if (Logger.is_installed("MPI - " + MPI_VERSION)): Logger.head("Skipping MPI") return Logger.start_installing("MPICH - " + MPI_VERSION) if not os.path.isfile(MPI_TAR_FILE): Utilities.download(" Downloading", MPI_URL, MPI_TAR_FILE) Utilities.validate_md5(" Validating MD5 sum", MPI_TAR_FILE, MPI_MD5SUM) Utilities.untar(" Extracting", MPI_TAR_FILE) if os.path.isdir(MPI_INSTALLER_PATH): env = self.env.copy() env['CC'] = Parameters.CC env['CXX'] = Parameters.CXX env['FC'] = Parameters.FC if not system.command_exists(env['CC'], env['PATH']): Logger.error('CC="' + env['CC'] + '" not found.') system.exit('COMPILER_NOT_FOUND') if not system.command_exists(env['CXX'], env['PATH']): Logger.error('CXX="' + env['CXX'] + '" not found.') system.exit('COMPILER_NOT_FOUND') config_options = [] if not system.command_exists(env['FC'], env['PATH']): Logger.error(' Warning: FC=' + env['FC'] + ' not found. MPICH will be compiled without fortran support.') config_options = ['--disable-fortran'] self.Install_library_with_configure(MPI_INSTALLER_PATH, library_env=env, max_lines=[3998, 7551, 3821], config_options=config_options) Logger.finish_installing("MPICH - " + MPI_VERSION) else: Logger.head("MPICH installer not found.") def Install_fftw(self): FFTW_VERSION="3.3.4" FFTW_TAR_FILE="fftw-3.3.4.tar.gz" FFTW_URL="http://turbulencehub.org/wp-content/uploads/Download_Files/fftw-3.3.4.tar.gz" FFTW_INSTALLER_PATH="fftw-3.3.4" FFTW_MD5SUM="2edab8c06b24feeb3b82bbb3ebf3e7b3" if (not Logger.is_installed("FFTW - " + FFTW_VERSION)) or (not os.path.isdir(FFTW_INSTALLER_PATH)): Logger.start_installing("FFTW - " + FFTW_VERSION) if not os.path.isfile(FFTW_TAR_FILE): Utilities.download(" Downloading", FFTW_URL, FFTW_TAR_FILE) Utilities.validate_md5(" Validating MD5 sum", FFTW_TAR_FILE, FFTW_MD5SUM) if os.path.isfile(FFTW_TAR_FILE): Utilities.untar(" Extracting", FFTW_TAR_FILE) if os.path.isdir(FFTW_INSTALLER_PATH): env = self.env.copy() env['CC'] = Parameters.MPICC env['CXX'] = Parameters.MPICXX if not system.command_exists(env['CC'], env['PATH']): Logger.error('MPICC="' + env['CC'] + '" not found.') system.exit('COMPILER_NOT_FOUND') if not system.command_exists(env['CXX'], env['PATH']): Logger.error('MPICXX="' + env['CXX'] + '" not found.') system.exit('COMPILER_NOT_FOUND') self.Install_library_with_configure(FFTW_INSTALLER_PATH, library_env=env, max_lines=[3998, 7551, 3821],config_options=['--enable-mpi'], message=" Installing with DOUBLE") self.Install_library_with_configure(FFTW_INSTALLER_PATH, library_env=env, max_lines=[3998, 7551, 3821],config_options=['--enable-mpi', '--enable-float'], message=" Installing with FLOAT") Logger.finish_installing("FFTW - " + FFTW_VERSION) else: Logger.head("Skipping FFTW") def Install_fftk(self): FFTK_VERSION="1.1" FFTK_TAR_FILE="fftk.tar.bz2" FFTK_URL="http://turbulencehub.org/wp-content/uploads/Download_Files/fftk.tar.bz2" FFTK_INSTALLER_PATH="fftk/trunk" FFTK_MD5SUM="f15ecaf6c250128a798e087a511dc2d9" if (not Logger.is_installed("FFTK - " + FFTK_VERSION)) or (not os.path.isdir(FFTK_INSTALLER_PATH)): Logger.start_installing("FFTK - " + FFTK_VERSION) if not os.path.isfile(FFTK_TAR_FILE): Utilities.download(" Downloading", FFTK_URL, FFTK_TAR_FILE) Utilities.validate_md5(" Validating MD5 sum", FFTK_TAR_FILE, FFTK_MD5SUM) if os.path.isfile(FFTK_TAR_FILE): Utilities.untar(" Extracting", FFTK_TAR_FILE) if os.path.isdir(FFTK_INSTALLER_PATH): env = self.env.copy() env['CC'] = Parameters.MPICC env['CXX'] = Parameters.MPICXX if not system.command_exists(env['CC'], env['PATH']): Logger.error('MPICC="' + env['CC'] + '" not found.') system.exit('COMPILER_NOT_FOUND') if not system.command_exists(env['CXX'], env['PATH']): Logger.error('MPICXX="' + env['CXX'] + '" not found.') system.exit('COMPILER_NOT_FOUND') self.Install_library_with_cmake(FFTK_INSTALLER_PATH, library_env=env, max_lines=[14, 9, 9], config_options=['-DREAL=DOUBLE'], message=" Installing with DOUBLE") self.Install_library_with_cmake(FFTK_INSTALLER_PATH, library_env=env, max_lines=[14, 9, 9], config_options=['-DREAL=FLOAT'] , message=" Installing with FLOAT") Logger.finish_installing("FFTK - " + FFTK_VERSION) else: Logger.head("Skipping FFTK") def Install_hdf5(self): HDF5_VERSION="1.8.20" HDF5_TAR_FILE="hdf5-1.8.20.tar.bz2" HDF5_URL="http://turbulencehub.org/wp-content/uploads/Download_Files/hdf5-1.8.20.tar.bz2" HDF5_INSTALLER_PATH="hdf5-1.8.20" HDF5_MD5SUM="23078d57975903e9536d1e7b299cc39c" if (not Logger.is_installed("HDF5 - " + HDF5_VERSION)) or (not os.path.isdir(HDF5_INSTALLER_PATH)): Logger.start_installing("HDF5 - " + HDF5_VERSION) if not os.path.isfile(HDF5_TAR_FILE): Utilities.download(" Downloading", HDF5_URL, HDF5_TAR_FILE) Utilities.validate_md5(" Validating MD5 sum", HDF5_TAR_FILE, HDF5_MD5SUM) if os.path.isfile(HDF5_TAR_FILE): Utilities.untar(" Extracting", HDF5_TAR_FILE) if os.path.isdir(HDF5_INSTALLER_PATH): env = self.env.copy() env['CC'] = Parameters.MPICC env['CXX'] = Parameters.MPICXX if not system.command_exists(env['CC'], env['PATH']): Logger.error('MPICC="' + env['CC'] + '" not found.') system.exit('COMPILER_NOT_FOUND') if not system.command_exists(env['CXX'], env['PATH']): Logger.error('MPICXX="' + env['CXX'] + '" not found.') system.exit('COMPILER_NOT_FOUND') self.Install_library_with_configure(HDF5_INSTALLER_PATH, library_env=env, max_lines=[463, 532, 258], config_options=['--enable-parallel']) Logger.finish_installing("HDF5 - " + HDF5_VERSION) else: Logger.head("Skipping HDF5") def Install_h5si(self): H5SI_VERSION="1.1" H5SI_TAR_FILE="h5si.tar.bz2" H5SI_URL="http://turbulencehub.org/wp-content/uploads/Download_Files/h5si.tar.bz2" H5SI_INSTALLER_PATH="h5si/trunk" H5SI_MD5SUM="eb32fa258aa137b530d171ed6b174939" if (not Logger.is_installed("H5SI - " + H5SI_VERSION)) or (not os.path.isdir(H5SI_INSTALLER_PATH)): Logger.start_installing("H5SI - " + H5SI_VERSION) if not os.path.isfile(H5SI_TAR_FILE): Utilities.download(" Downloading", H5SI_URL, H5SI_TAR_FILE) Utilities.validate_md5(" Validating MD5 sum", H5SI_TAR_FILE, H5SI_MD5SUM) if os.path.isfile(H5SI_TAR_FILE): Utilities.untar(" Extracting", H5SI_TAR_FILE) if os.path.isdir(H5SI_INSTALLER_PATH): env = self.env.copy() env['CC'] = Parameters.MPICC env['CXX'] = Parameters.MPICXX if not system.command_exists(env['CC'], env['PATH']): Logger.error('MPICC="' + env['CC'] + '" not found.') system.exit('COMPILER_NOT_FOUND') if not system.command_exists(env['CXX'], env['PATH']): Logger.error('MPICXX="' + env['CXX'] + '" not found.') system.exit('COMPILER_NOT_FOUND') self.Install_library_with_cmake(H5SI_INSTALLER_PATH, library_env=env, max_lines=[17, 21, 17]) Logger.finish_installing("H5SI - " + H5SI_VERSION) else: Logger.head("Skipping H5SI") def main(): if len(sys.argv) > 2: Logger.log('Please provide only one argument to this program.') return if len(sys.argv) == 2: if sys.argv[1] == '-h' or sys.argv[1] == '--help': Utilities.help() system.exit('SUCCESS') elif sys.argv[1] == '-d' or sys.argv[1] == '--default': Parameters.SELECT_DEFAULT = True else: Logger.log('Invalid parameter: ' + sys.argv[1]) if Parameters.LOG.lower() == 'yes': Logger.enable_file_logging() Logger.log_parameters() if not Parameters.SELECT_DEFAULT: Logger.log('') Logger.log('Leave the following options blank to the select the [default value]:') # Ask installation directory (PREFIX) Logger.log('') Parameters.PREFIX = Utilities.ask("Installation path [%s]: " % Parameters.PREFIX, Parameters.PREFIX) #Ask which libraries to install (OPTIONS) Logger.log('') Logger.log('Please select the libraries you want to install:') Logger.log('1) CMAKE') Logger.log('2) Blitz++') Logger.log('3) YAML-CPP') Logger.log('4) MPICH') Logger.log('5) FFTW') Logger.log('6) FFTK') Logger.log('7) HDF5') Logger.log('8) H5SI') Logger.log('9) All') Logger.log('10) Exit') Logger.log('') Logger.log('If you want to install multiple libraries, enter values seperated by space.') Logger.log('For example: 1 4 6') Logger.log('Note: This script skip libraries for which the installer folder pre-exists.') Parameters.OPTIONS = Utilities.ask('Enter your choice [%s]: ' % Parameters.OPTIONS, Parameters.OPTIONS) if (Parameters.OPTIONS == '10'): system.exit('SUCCESS') # Ask whether RC_FILE be updated automatically (UPDATE_RC_FILE) Logger.log('') Parameters.UPDATE_RC_FILE = Utilities.ask('Do you want "%s" be updated automatically (yes/no) ? [%s]: ' % (Parameters.RC_FILE, Parameters.UPDATE_RC_FILE), Parameters.UPDATE_RC_FILE) # Ask how many cores to use for parallel make (J) J = Parameters.J if J == 'None': # the script has not detect it J = 1 Logger.log('') J = Utilities.ask('How many cores to use for parallel make ? [%s]: ' % (J), J) if (Parameters.J != 'None') and (int(J) > system.available_cpu_count()): Logger.error('Warning: supplied value of J=%s is greater than number of cores available = %s, therefore using J=%s.' % (J, str(system.available_cpu_count()), str(system.available_cpu_count()))) Parameters.J = str(system.available_cpu_count()) else: Parameters.J = J installers = Installers() Parameters.OPTIONS = Parameters.OPTIONS.split() if ('1' in Parameters.OPTIONS) or ('9' in Parameters.OPTIONS): installers.Install_cmake() if ('2' in Parameters.OPTIONS) or ('9' in Parameters.OPTIONS): installers.Install_blitz() if ('3' in Parameters.OPTIONS) or ('9' in Parameters.OPTIONS): installers.Install_yaml() if ('4' in Parameters.OPTIONS) or ('9' in Parameters.OPTIONS): installers.Install_mpi() if ('5' in Parameters.OPTIONS) or ('9' in Parameters.OPTIONS): installers.Install_fftw() if ('6' in Parameters.OPTIONS) or ('9' in Parameters.OPTIONS): installers.Install_fftk() if ('7' in Parameters.OPTIONS) or ('9' in Parameters.OPTIONS): installers.Install_hdf5() if ('8' in Parameters.OPTIONS) or ('9' in Parameters.OPTIONS): installers.Install_h5si() # Update 'RC_FILE', if UPDATE_RC_FILE = yes # Otherwise display these variables on stdout if Parameters.UPDATE_RC_FILE.lower() == 'yes': Logger.log('') Logger.log('Following lines have been added to "%s".' % (Parameters.RC_FILE)) # Append these to RC_FILE and LOG_FILE Logger.add_file(Parameters.RC_FILE) Logger.log('') Logger.log('#---------------------------------------------------------------------------') Logger.log('#Added by -- Tarang Dependency Installer -- on %s' % time.strftime("%d/%m/%Y", time.gmtime())) Logger.log('#---------------------------------------------------------------------------\n') Logger.log('export PATH=%s/bin:$PATH' % Parameters.PREFIX) Logger.log('export PKG_CONFIG_DISABLE_UNINSTALLED=true') Logger.log('export PKG_CONFIG_PATH=%s/lib/pkgconfig:$PKG_CONFIG_PATH' % Parameters.PREFIX) Logger.log('export HDF5_ROOT=%s' % Parameters.PREFIX) Logger.log('export CPATH=%s/include/:$CPATH' % Parameters.PREFIX) Logger.log('export LD_LIBRARY_PATH=%s/lib:$LD_LIBRARY_PATH' % Parameters.PREFIX) Logger.log('export LIBRARY_PATH=%s/lib:$LIBRARY_PATH' % Parameters.PREFIX) Logger.log('export MANPATH=%s/share/man/:$MANPATH' % Parameters.PREFIX) Logger.log('#---------------------------------------------------------------------------') Logger.log('') Logger.remove_file(Parameters.RC_FILE) else: Logger.log('') Logger.log('---------------------------------------------------------------------------') Logger.log('Export these environment variables to compile tarang, or add them to "%s"' % Parameters.RC_FILE) Logger.log('---------------------------------------------------------------------------') Logger.log('') Logger.log('export PATH=%s/bin:$PATH' % Parameters.PREFIX) Logger.log('export PKG_CONFIG_DISABLE_UNINSTALLED=true') Logger.log('export PKG_CONFIG_PATH=%s/lib/pkgconfig:$PKG_CONFIG_PATH' % Parameters.PREFIX) Logger.log('export HDF5_ROOT=%s' % Parameters.PREFIX) Logger.log('export CPATH=%s/include/:$CPATH' % Parameters.PREFIX) Logger.log('export LD_LIBRARY_PATH=%s/lib:$LD_LIBRARY_PATH' % Parameters.PREFIX) Logger.log('export LIBRARY_PATH=%s/lib:$LIBRARY_PATH' % Parameters.PREFIX) Logger.log('export MANPATH=%s/share/man/:$MANPATH' % Parameters.PREFIX) if __name__ == '__main__': main()