HEX
Server: nginx/1.30.4
System: Linux d-twindigital-web-1.host-ed.eu 4.18.0-553.141.2.el8_10.x86_64 #1 SMP Wed Jul 8 10:28:18 EDT 2026 x86_64
User: u_twindigital_html (1009)
PHP: 8.3.30
Disabled: NONE
$ pwd: /lib/python3.6/site-packages/S3/__pycache__/
Upload Files
File: //lib/python3.6/site-packages/S3/__pycache__/FileLists.cpython-36.pyc
3

@tJ[Cd�@sddlmZddlmZddlmZddlmZddlmZddlTddlmZdd	l	m	Z	dd
l
mZmZm
Z
ddlZddlZddlZddlZddlZddlZdd
dgZdd�Zdd�Zdd�Zdd�Zdd�Zdd�Zddd�Zddifdd
�Zdd�ZdS) �)�absolute_import�)�S3)�Config)�S3Uri)�FileDict)�*)�ParameterError)�	HashCache)�debug�info�warningN�fetch_local_list�fetch_remote_list�compare_filelistsccs�ytjt|��}Wn
dSgg}}xN|D]F}t|�}tjjttjj||���rjt||�st|j|�q.|j|�q.W|||fVxB|D]:}tjj||�}tjj	t|��s�xt
|�D]
}|Vq�Wq�WdS)z_
    Reimplementation of python's os.walk to nicely support unicode in input as in output.
    N)�os�listdir�deunicodise�	unicodise�path�isdir�join�handle_exclude_include_walk_dir�append�islink�_os_walk_unicode)�top�names�dirs�nondirs�name�new_path�x�r#�/usr/lib/python3.6/FileLists.pyrs"



rcCs�t�}tjj||d�}td|�d}xJ|jD]@}|jjd�pH|jjd�sLq.|j|�r.d}td|j	|�Pq.W|r�x\|j
D]R}|jjd�p�|jjd�s�q~td||jf�|j|�r~d}td	|j|�Pq~W|r�td
|�ntd|�|S)a

    Should this root/dirname directory be excluded? (otherwise included by default)
    Exclude dir matches in the current directory
    This prevents us from recursing down trees we know we want to ignore
    return True for including, and False for excluding
    �z	CHECK: %rFz\/$z	\/\Z(?ms)TzEXCL-MATCH: '%s'zINCL-TEST: %s ~ %szINCL-MATCH: '%s'zEXCLUDE: %rzPASS: %r)rrrrr�exclude�pattern�endswith�search�
debug_exclude�include�
debug_include)�root�dirname�cfg�d�excluded�rr#r#r$r7s0

rc
cs�tjjt|��st�t|g�}x�t|�D]�\}}}ttjjt|���}xp|D]h}tjj	||�}ttjjt|���}tjj
t|��rL||ks�|j|tjj�r�t
d|�qL|j|�qLWq(Wx0|D](}	x"t|	�D]\}}}|||fVq�Wq�WdS)z�
    Walk filesystem, following symbolic links (but without recursion), on python2.4 and later

    If a symlink directory loop is detected, emit a warning and skip.
    E.g.: dir1/dir2/sym-dir -> ../dir2
    z+Skipping recursively symlinked directory %sN)rrrr�AssertionError�setrr�realpathrr�
startswith�sepr
�add)
rZwalkdirs�dirpath�dirnames�	filenamesZreal_dirpathr.�currentZreal_currentZwalkdirr#r#r$�_fswalk_follow_symlinksZs


r=ccs(x"t|�D]\}}}|||fVq
WdS)z\
    Directory tree generator

    path (str) is the root of the directory tree to walk
    N)r)rr9r:r;r#r#r$�_fswalk_no_symlinksrsr>cCs�td�t�}tdd�}x�|j�D]�}td|�d}x0|jD]&}|j|�r>d}td|j|�Pq>W|r�x0|jD]&}|j|�rtd}td|j|�PqtW|r�td|�||||<||=q"q"td	|�q"W||fS)
NzApplying --exclude/--includeF)�ignore_casez	CHECK: %sTzEXCL-MATCH: '%s'zINCL-MATCH: '%s'zEXCLUDE: %szPASS: %r)	rrr�keysr&r)r*r+r,)�src_listr/�exclude_list�filer1r2r#r#r$�filter_exclude_include{s0


rDcCsPdd�}i}x�|jD]�}z�d}|dkr>tjtjj�ddd�}nNytjt|�dd�}Wn6tk
r�}ztd||j	f�wWYdd}~XnXxb|D]Z}t
|�j�}tj
jtj
j||��}t
tj
jt|���}t
tj
jt|���}	||||	�q�WWd|�r|j�XqWg}
|j�}|j�x.|D]&}||}
|
j�|
j|g|
f��q"W|
S)	NcSs&||kr|g||<n||j|�dS)N)r)r0�key�valuer#r#r$�_append�sz(_get_filelist_from_file.<locals>._append�-r2F)�mode�closefd)rIzJ--files-from input file %s could not be opened for reading (%s), skipping.)�
files_from�io�open�sys�stdin�filenor�IOErrorr
�strerrorr�striprr�normpathrr.�basename�closer@�sortr)r/�
local_pathrG�filelistZfname�f�e�liner.rU�resultr@rE�valuesr#r#r$�_get_filelist_from_file�s8

r_FcsD��fdd�}��fdd�}�fdd�}t��t���jrZtjjt�j��rZ�j�j�ntd�g}t	dd	�}d}t
|�ttt
fkr�|g}|dkr��j}xN|D]F}	t|	�}
|
j
d
ks�td|	��|
j�r�|r�td|	��|j|
�q�Wx|D]}
|||
��\}}q�Wt|�d
k�rd}t|�\}}||�}
|�|�||||
fS)Nc	stt|�}d}td|�d}�xP|D�]F}|d7}|ddkrNtd||f�|dkrXq$||d}ytjtjt|���}Wn4tk
r�}z|jtjkr�w$n�WYdd}~XnX||j	|j
|j|j|j
|j|j|d��||j
7}d	�jkr$�j|j|j
|j|j
�}|dk�rTy|j|�}Wntk
�r8w$YnX�j|j|j
|j|j
|�|j||j|j
||j
�q$W|S)
NrzYRunning stat() and reading/calculating MD5 values on %d files, this may take some time...ri�z[%d/%d]rH�	full_name)�size�mtime�dev�inode�uid�gid�sr�md5)�lenrr�stat_result�statr�OSError�errno�ENOENT�update�st_size�st_mtime�st_dev�st_ino�st_uid�st_gid�sync_checksrh�get_md5rQr8Zrecord_hardlink)	�loc_listZlen_loc_list�
total_sizeZcounter�
relative_filer`rgr[rh)�cacher/r#r$�_fetch_local_list_info�sH



z0fetch_local_list.<locals>._fetch_local_list_infocs�td�|j�dkr^ytj�}tj�}Wnd}d}YnXddd||ddd�|d<|dfS|j�r�|j�}|j�}�r�t�j�r�t	�|�}d}qԈj
r�t|�}nt|�}d}n"d}|j
�}|g|j�gfg}d}�x�|D]�\}	}
}|	j||d�}x�|D]�}
tjj|	|
�}tjjt|���s<tjjt|��r�td	|�q�tjjt|���rd�j
�sdtd
|�q�tjj||
�}tjjdk�r�dj|jtjj��}�jdk�r�t|�}|jd
��r�|dd�}d|i||<q�Wq�W||fS)Nz Compiling list of local files...rHrr)r`rarbrerfrcrdTFr%z"Skipping over non regular file: %szSkipping over symbolic link: %s�/Znormalz./�r`���r)rrUr�geteuid�getegidrrrirKr_�follow_symlinksr=r>r.�replacer�isfiler�existsr
rr7�splitZurlencoding_modeZreplace_nonprintablesr6)rxZ	local_urir{rerfZ
local_baserXrY�single_filer-r�filesZrel_rootrZr`rz)r/�is_srcr#r$�_get_filelist_local�sf



z-fetch_local_list.<locals>._get_filelist_localcsv�jrrt�j�dkrr|j�x@|j�D]4}|j||d||d||d||d�q&W|j�|j�j�dS)Nrrcrdrbra)�
cache_filerirKZmark_all_for_purger@Zunmark_for_purge�purge�save)r{�
local_list�i)r/r#r$�_maintain_cache.s4z)fetch_local_list.<locals>._maintain_cachez!No cache file found, creating it.F)r?rCz.Expecting filename or directory instead of: %sz)Use --recursive to upload a directory: %sr)rr
r�rrr�Z
deunicodise_s�loadrr�type�list�tupler4�	recursiverr	rrrirD)�argsr�r�r|r�r�Z
local_urisr�r��arg�uriZlist_for_urirBryr#)r{r/r�r$r�s<,?




cs �fdd��d��fdd�	}t��g}tdd�}t|�tttfkrH|g}|dkrV�j}x4|D],}t|�}|jdks~td	|��|j	|�q\Wd
}	|r�xP|D]H}||dd�\}
}|	|7}	x*|
D]"}|
|||<|j
||
j|��q�Wq�W�n �x|D�]}|j�}
t
jd|
d
d�}t|�dk�r�|\}}d|k}|t|�|d�\}
}|	|7}	x�|
D]*}tjj|
|d|
��rP|
|||<�qPWq�ttjjt|j����}|�s�td|j���||j�|j�d�}|�r҈||�|||<|jd�}|�r�|j
||�|	|jdd
�7}	q�Wt|�\}}|||	fS)Ncs�t��j|�}|jd�sdS|jt|dd�|ddjd�t|dd�d��y*|dd}|jd|i�td	|�Wntk
r�YnXdS)
NZheaderszcontent-lengthZetagz"'z
last-modified)rarh�	timestampzs3cmd-attrsrhzretreived md5=%s from headers)	rZobject_info�getro�intrSZdateRFC822toUnixr�KeyError)r��remote_item�responserh)r/r#r$�_get_remote_attribses
z.fetch_remote_list.<locals>._get_remote_attribsTc
s�td|�tjd�}d}tt��}|j|j�|j�|�d�}|j�}}|}|dkr�|ddkr�|d|jd�d�}t	d|j�|f�}t
|�}	td	d
�}
d	}�x|dD�]}|d|ko�|dddk�rtt
jjt|d���}
|j�}d
}td	d
�}
n|d|	d�}
|j�|
}|j|
��r4td�q�t|d�t|d�|djd�|d||ddd�|
|
<d|
|
dk�r��t	|�|
|
�|
|
d}|
j|
|�|t|d�7}|r�Pq�W|
|fS)Nz*Retrieving list of remote files for %s ...z\A\s*\Zr)�prefixr��
uri_paramsr%rr}z
s3://%s/%sF)r?r�ZKeyTz(Empty object name on S3 found, ignoring.ZSizeZLastModifiedZETagz"')rar�rh�
object_key�object_uri_str�base_urircrdrHrhrr)r�re�compilerrZbucket_listZbucket�object�rfindrrirrrrrUrr��matchr
r�ZdateS3toUnixrS�
record_md5)Z
remote_urir�Zempty_fname_rery�s3r�Zrem_base_originalZrem_baseZremote_uri_originalZrem_base_lenZrem_listZ	break_nowr�rEr�rh)r�r�r#r$�_get_filelist_remotevsR





z/fetch_remote_list.<locals>._get_filelist_remoteF)r?r�z Expecting S3 URI instead of '%s'r)r�z\*|\?r)�maxsplitr~r}r�z3Expecting S3 URI with a filename or --recursive: %s)r�r�r�rhra)T)rrr�r�r�r4r�rr	rr�rwr�r�r�ri�glob�fnmatchrrrrUrr�r�rD)r�Zrequire_attribsr�r�r�Zremote_urisZremote_listr�r�ryZ
objectlistZtmp_total_sizerEZuri_strZwildcard_split_resultr��restZneed_recursionr�rhrBr#)r�r/r�r$rds`B







c
srdd�}��fdd�}|dko$|dks,t�td�t��tdd�}g}td||�||�f��x�|j�D�]�}td	|�|�k�r��jr�td
|�||=�|=qly||�|||�}	Wn>ttfk
r�td|�t	d|�||=�|=wlYnX|	�rtd
|�||=�|=n�y|j
|�}
Wntk
�rDd}
YnX|
dk	�r�|
�jk�r��j|
�}td||f�|j
||||f�||=�|=n$�j||
�||||<||=�|=qly|j
|�}
Wntk
�r�d}
YnX�j|
�}|dk	�r(td||f�|j
||||f�||=ql�j||
�qlWx,�j�D] }||k�sZ||k�rB�|=�qBW|�||fS)NcSs|rdp
dS)NZremote�localr#)Z	is_remoter#r#r$�__direction_str�sz*compare_filelists.<locals>.__direction_strc	
s\d}||ko|�ks2td|||k|�kf�dSd�jkr�d�|kr�d||kr��|d||dkr�td|||d�|df�d}d�jk}|r�|dkr�d||dks�|dkr�d�|dkr�d}td|�|o�|�rXy|j|�}�j|�}Wn4ttfk
�r6td	|�td
|��YnX||k�rXd}td|||f�|S)z@Return True if src_list[file] matches dst_list[file], else FalseTzE%s: does not exist in one side or the other: src_list=%s, dst_list=%sFraz'xfer: %s (size mismatch: src=%s dst=%s)rhrHzdisabled md5 check for %szIGNR: %s (disappeared)z%s: file disappeared, ignoring.z&XFER: %s (md5 mismatch: src=%s dst=%s))rrvrrwrQrlr
)	rAZdst_lst�
src_remote�
dst_remoterCZ
attribs_matchZcompare_md5Zsrc_md5Zdst_md5)r/�dst_listr#r$�_compare�s4
"
0


z#compare_filelists.<locals>._compareFzVerifying attributes...)r?z)Comparing filelists (direction: %s -> %s)z	CHECK: %szIGNR: %s (used --skip-existing)zIGNR: %s (disappeared)z%s: file disappeared, ignoring.zIGNR: %s (transfer not needed)zDST COPY src: %s -> %szDST COPY dst: %s -> %s)r3rrrrr@Z
skip_existingrQrlr
rwZby_md5Zfind_md5_onerr�)
rAr�r�r�r�r�Zupdate_listZ
copy_pairsrzZ	same_filerhZdst1rZr#)r/r�r$r�sp'






)FN)Z
__future__rrrrrZUtilsZ
Exceptionsr	r
�loggingrrr
rrNr�r�rmrL�__all__rrr=r>rDr_rrrr#r#r#r$�<module>	s4
#	(
$
@ Telegram