file_inspector Namespace Reference

Functions
	argument_parser ()
	add_failed_check (inspection_check_id, file_guid, error_type, error_details, conn)
	transfer_file (source_cluster, source_storage, source_file_path, source_xrootd_url, source_file_size, source_file_checksum, destination_cluster, destination_storage, destination_file_path, destination_xrootd_url, transfer_protocol_code, source_ssh_prefix, destination_ssh_prefix, start_tmp_dir, conn)
	check_files (file_list, catalogue_files, cluster_name, storage_name, xrootd_url, access_protocol, ssh_prefix, full_check, catalogue_matches, start_tmp_dir, conn, inspection_check_id)
	define_transfer_protocol (so_transfer_protocol, si_transfer_protocol, oi_transfer_protocol, io_transfer_protocol)
	compare_storages (inspect_cluster, inspect_storage, inspect_storage_path, inspect_xrootd_url, origin_cluster, origin_storage, origin_storage_path, origin_xrootd_url, replica_flags, start_cluster, start_tmp_dir, si_check_protocol, si_transfer_protocol, si_ssh_prefix, conn)
	send_email (from_address, mail_address, subject, body)

Variables
str	mail_address = "gertsen@jinr.ru"
str	function_file_path = os.path.dirname(os.path.realpath(__file__)) + "/../function_set.py"
	code = compile(source_file.read(), function_file_path, "exec")
	config = json.load(open("file_inspector.json"))
	conn = psycopg2.connect(("dbname=%s user=%s host=%s password=%s") % (config["db_name"], config["db_user"], config["db_host"], config["db_pass"]))
	cursor = conn.cursor()
	cursor_result = cursor.fetchall()
	cluster_dictionary = list(chain(*cursor_result))
list	storage_dictionary_expanded = ["*"]
list	storage_dictionary = []
int	cnt_all_files = 0
	start_cluster
	inspect_storage_address
	storage_2cron_address
	cluter_host = socket.gethostname()
	start_tmp_dir
	start_ssh_prefix
	current_script_path = os.path.realpath(__file__)
str	storage_2cron_cluster = "*"
str	storage_2cron_storage = "*"
	storage_2cron_array = storage_2cron_address.split(":")
str	storage_request = "select s.check_interval,s.cluster_name,s.storage_name from storage_ s "
	storage_2cron_list = cursor.fetchall()
	process_output = process_output.strip()
	process_error
	process_returncode
	inspect_storage_array = inspect_storage_address.split(":")
	inspect_cluster = inspect_storage_array[0]
	inspect_storage = inspect_storage_array[1]
	inspection_starttime = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
	inspect_storage_path
	inspect_storage_type
	inspect_xrootd_url
	inspect_storage_interval
	inspect_storage_full
	check_flags
	replica_cluster
	replica_storage
	replica_flags
	inspect_ssh_prefix
	si_check_protocol
	si_transfer_protocol
bool	full_check = False
	catalogue_files = cursor.fetchall()
list	catalogue_matches = []
	catalogue_file_count = len(catalogue_files)
	inspection_check_id = cursor.fetchone()[0]
	all_files = glob.glob(inspect_storage_path + '/**/*', recursive=True)
bool	ssh_failed = True
	returncode
	xrd_file_list = get_files_xrd(inspect_storage_path, inspect_xrootd_url)
	eos_file_list = get_files_eos(inspect_storage_path, inspect_xrootd_url)
tuple	delete_absent_files = ("d" in check_flags)
	catalogue_file_guid
	catalogue_file_path
	_
	replica_storage_path
	replica_xrootd_url
	inspection_endtime = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
	found_errors = cursor.fetchone()[0]
str	from_address = "<inspector@jinr.ru>"
str	subject = "File Inspector - Integrity Results"
tuple	body

Function Documentation

add_failed_check()

file_inspector.add_failed_check	(		inspection_check_id,
			file_guid,
			error_type,
			error_details,
			conn
	)

Definition at line 116 of file file_inspector.py.

Referenced by check_files().

argument_parser()

file_inspector.argument_parser

(

)

Definition at line 63 of file file_inspector.py.

check_files()

file_inspector.check_files	(		file_list,
			catalogue_files,
			cluster_name,
			storage_name,
			xrootd_url,
			access_protocol,
			ssh_prefix,
			full_check,
			catalogue_matches,
			start_tmp_dir,
			conn,
			inspection_check_id
	)

Definition at line 258 of file file_inspector.py.

References add_failed_check().

compare_storages()

file_inspector.compare_storages	(		inspect_cluster,
			inspect_storage,
			inspect_storage_path,
			inspect_xrootd_url,
			origin_cluster,
			origin_storage,
			origin_storage_path,
			origin_xrootd_url,
			replica_flags,
			start_cluster,
			start_tmp_dir,
			si_check_protocol,
			si_transfer_protocol,
			si_ssh_prefix,
			conn
	)

Definition at line 400 of file file_inspector.py.

References define_transfer_protocol(), and transfer_file().

define_transfer_protocol()

file_inspector.define_transfer_protocol	(		so_transfer_protocol,
			si_transfer_protocol,
			oi_transfer_protocol,
			io_transfer_protocol
	)

Definition at line 368 of file file_inspector.py.

Referenced by compare_storages().

send_email()

file_inspector.send_email	(		from_address,
			mail_address,
			subject,
			body
	)

Definition at line 523 of file file_inspector.py.

transfer_file()

file_inspector.transfer_file	(		source_cluster,
			source_storage,
			source_file_path,
			source_xrootd_url,
			source_file_size,
			source_file_checksum,
			destination_cluster,
			destination_storage,
			destination_file_path,
			destination_xrootd_url,
			transfer_protocol_code,
			source_ssh_prefix,
			destination_ssh_prefix,
			start_tmp_dir,
			conn
	)

Definition at line 125 of file file_inspector.py.

Referenced by compare_storages().

Variable Documentation

_

file_inspector._

protected

Definition at line 714 of file file_inspector.py.

all_files

list file_inspector.all_files = glob.glob(inspect_storage_path + '/**/*', recursive=True)

Definition at line 674 of file file_inspector.py.

body

tuple file_inspector.body

Initial value:

=  (f"File inspection on '{inspect_storage_address}' storage completed\n"
        f"Operation time: {inspection_starttime} - {inspection_endtime}\n"
        f"Number of errors: {found_errors}\n")

Definition at line 753 of file file_inspector.py.

catalogue_file_count

file_inspector.catalogue_file_count = len(catalogue_files)

Definition at line 665 of file file_inspector.py.

catalogue_file_guid

file_inspector.catalogue_file_guid

Definition at line 714 of file file_inspector.py.

catalogue_file_path

file_inspector.catalogue_file_path

Definition at line 714 of file file_inspector.py.

catalogue_files

file_inspector.catalogue_files = cursor.fetchall()

Definition at line 660 of file file_inspector.py.

catalogue_matches

list file_inspector.catalogue_matches = []

Definition at line 661 of file file_inspector.py.

check_flags

file_inspector.check_flags

Definition at line 627 of file file_inspector.py.

cluster_dictionary

file_inspector.cluster_dictionary = list(chain(*cursor_result))

Definition at line 46 of file file_inspector.py.

cluter_host

file_inspector.cluter_host = socket.gethostname()

Definition at line 561 of file file_inspector.py.

cnt_all_files

int file_inspector.cnt_all_files = 0

Definition at line 59 of file file_inspector.py.

code

file_inspector.code = compile(source_file.read(), function_file_path, "exec")

Definition at line 30 of file file_inspector.py.

config

file_inspector.config = json.load(open("file_inspector.json"))

Definition at line 34 of file file_inspector.py.

conn

file_inspector.conn = psycopg2.connect(("dbname=%s user=%s host=%s password=%s") % (config["db_name"], config["db_user"], config["db_host"], config["db_pass"]))

Definition at line 36 of file file_inspector.py.

current_script_path

file_inspector.current_script_path = os.path.realpath(__file__)

Definition at line 583 of file file_inspector.py.

cursor

file_inspector.cursor = conn.cursor()

Definition at line 40 of file file_inspector.py.

cursor_result

file_inspector.cursor_result = cursor.fetchall()

Definition at line 45 of file file_inspector.py.

delete_absent_files

tuple file_inspector.delete_absent_files = ("d" in check_flags)

Definition at line 711 of file file_inspector.py.

eos_file_list

file_inspector.eos_file_list = get_files_eos(inspect_storage_path, inspect_xrootd_url)

Definition at line 704 of file file_inspector.py.

found_errors

file_inspector.found_errors = cursor.fetchone()[0]

Definition at line 745 of file file_inspector.py.

from_address

str file_inspector.from_address = "<inspector@jinr.ru>"

Definition at line 751 of file file_inspector.py.

full_check

bool file_inspector.full_check = False

Definition at line 644 of file file_inspector.py.

function_file_path

str file_inspector.function_file_path = os.path.dirname(os.path.realpath(__file__)) + "/../function_set.py"

Definition at line 28 of file file_inspector.py.

inspect_cluster

file_inspector.inspect_cluster = inspect_storage_array[0]

Definition at line 616 of file file_inspector.py.

inspect_ssh_prefix

file_inspector.inspect_ssh_prefix

Definition at line 627 of file file_inspector.py.

inspect_storage

file_inspector.inspect_storage = inspect_storage_array[1]

Definition at line 617 of file file_inspector.py.

inspect_storage_address

file_inspector.inspect_storage_address

Definition at line 557 of file file_inspector.py.

inspect_storage_array

file_inspector.inspect_storage_array = inspect_storage_address.split(":")

Definition at line 615 of file file_inspector.py.

inspect_storage_full

file_inspector.inspect_storage_full

Definition at line 627 of file file_inspector.py.

inspect_storage_interval

file_inspector.inspect_storage_interval

Definition at line 627 of file file_inspector.py.

inspect_storage_path

file_inspector.inspect_storage_path

Definition at line 627 of file file_inspector.py.

inspect_storage_type

file_inspector.inspect_storage_type

Definition at line 627 of file file_inspector.py.

inspect_xrootd_url

file_inspector.inspect_xrootd_url

Definition at line 627 of file file_inspector.py.

inspection_check_id

file_inspector.inspection_check_id = cursor.fetchone()[0]

Definition at line 668 of file file_inspector.py.

inspection_endtime

file_inspector.inspection_endtime = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")

Definition at line 738 of file file_inspector.py.

inspection_starttime

file_inspector.inspection_starttime = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")

Definition at line 619 of file file_inspector.py.

mail_address

str file_inspector.mail_address = "gertsen@jinr.ru"

Definition at line 22 of file file_inspector.py.

process_error

file_inspector.process_error

Definition at line 608 of file file_inspector.py.

process_output

file_inspector.process_output = process_output.strip()

Definition at line 608 of file file_inspector.py.

process_returncode

file_inspector.process_returncode

Definition at line 608 of file file_inspector.py.

replica_cluster

file_inspector.replica_cluster

Definition at line 627 of file file_inspector.py.

replica_flags

file_inspector.replica_flags

Definition at line 627 of file file_inspector.py.

replica_storage

file_inspector.replica_storage

Definition at line 627 of file file_inspector.py.

replica_storage_path

file_inspector.replica_storage_path

Definition at line 733 of file file_inspector.py.

replica_xrootd_url

file_inspector.replica_xrootd_url

Definition at line 733 of file file_inspector.py.

returncode

file_inspector.returncode

Definition at line 686 of file file_inspector.py.

si_check_protocol

file_inspector.si_check_protocol

Definition at line 641 of file file_inspector.py.

si_transfer_protocol

file_inspector.si_transfer_protocol

Definition at line 641 of file file_inspector.py.

ssh_failed

bool file_inspector.ssh_failed = True

Definition at line 684 of file file_inspector.py.

start_cluster

file_inspector.start_cluster

Definition at line 557 of file file_inspector.py.

start_ssh_prefix

file_inspector.start_ssh_prefix

Definition at line 576 of file file_inspector.py.

start_tmp_dir

file_inspector.start_tmp_dir

Definition at line 576 of file file_inspector.py.

storage_2cron_address

file_inspector.storage_2cron_address

Definition at line 557 of file file_inspector.py.

storage_2cron_array

file_inspector.storage_2cron_array = storage_2cron_address.split(":")

Definition at line 589 of file file_inspector.py.

storage_2cron_cluster

str file_inspector.storage_2cron_cluster = "*"

Definition at line 586 of file file_inspector.py.

storage_2cron_list

file_inspector.storage_2cron_list = cursor.fetchall()

Definition at line 602 of file file_inspector.py.

storage_2cron_storage

str file_inspector.storage_2cron_storage = "*"

Definition at line 587 of file file_inspector.py.

storage_dictionary

list file_inspector.storage_dictionary = []

Definition at line 54 of file file_inspector.py.

storage_dictionary_expanded

list file_inspector.storage_dictionary_expanded = ["*"]

Definition at line 47 of file file_inspector.py.

storage_request

str file_inspector.storage_request = "select s.check_interval,s.cluster_name,s.storage_name from storage_ s "

Definition at line 594 of file file_inspector.py.

subject

str file_inspector.subject = "File Inspector - Integrity Results"

Definition at line 752 of file file_inspector.py.

xrd_file_list

file_inspector.xrd_file_list = get_files_xrd(inspect_storage_path, inspect_xrootd_url)

Definition at line 696 of file file_inspector.py.