ich hoffe jmd kann mir helfen. Ich versuche in Azure Data Lake Storage Dateien mit einem Python Code aus einem lokalen Rechner zu kopieren. Zuvor habe ich lokal ein Testumfeld erstellt und dort funktioniert das kopieren. Auch die Connection mit Azure Data Lake Storage funktioniert, leider funktioniert das kopieren in Data Lake Storage nicht. mein Code sieht wie folgt aus(ich wurde mich auf eine Antwort freuen):
Code: Alles auswählen
## Use this for Azure AD authentication
from msrestazure.azure_active_directory import AADTokenCredentials
## Required for Azure Data Lake Storage Gen1 account management
from azure.mgmt.datalake.store import DataLakeStoreAccountManagementClient
from azure.mgmt.datalake.store.models import DataLakeStoreAccount
## Required for Azure Data Lake Storage Gen1 filesystem management
from azure.datalake.store import core, lib, multithread
# Common Azure imports
import adal
from azure.mgmt.resource.resources import ResourceManagementClient
from azure.mgmt.resource.resources.models import ResourceGroup
## Use these as needed for your application
import logging, pprint, uuid, time
import os, os.path
import shutil
def lake_auth():
### Authentification
adlCreds = lib.auth(tenant_id='', resource = 'https://datalake.azure.net/')
### ADL client
## Declare variables
subscriptionId = ''
adlsAccountName = ''
## Create a filesystem client object
adlsFileSystemClient = core.AzureDLFileSystem(adlCreds, store_name=adlsAccountName)
return adlsFileSystemClient
#Call auth function
lake = lake_auth()
### Logic
src = ''
dest = ''
years = ['2017', '2018']
src_folders = os.listdir(src)
for folder_name in src_folders:
folder_path = src+'\\'+folder_name
if ' ' in folder_path:
pass
else:
if '' in folder_path:
src_files = os.listdir(folder_path)
for file_name in src_files:
final_src = src + '\\' + folder_name + '\\' + file_name
for year in years:
for month in range(1, 12+1):
for day in range(1,31+1):
if len(str(month)) == 1 and len(str(day)) == 1:
if year+'-0'+str(month)+'-0'+str(day) in file_name:
if (os.path.isfile(final_src)):
final_dest = dest + '\\ \\' + folder_name + '\\' + str(year) + '\\' + str(month) + '\\' + str(day) + '\\' + file_name
shutil.copy(final_src, final_dest)
multithread.ADLUploader(lake, lpath=final_src, rpath=final_dest, nthreads=64, overwrite=True, buffersize=4194304, blocksize=4194304)
elif len(str(month)) == 1 and len(str(day)) == 2:
if year+'-0'+str(month)+'-'+str(day) in file_name:
if (os.path.isfile(final_src)):
final_dest = dest + '\\ \\' + folder_name + '\\' + str(year) + '\\' + str(month) + '\\' + str(day)
shutil.copy(final_src, final_dest)
multithread.ADLUploader(lake, lpath=final_src, rpath=final_dest, nthreads=64, overwrite=True, buffersize=4194304, blocksize=4194304)
else:
if year+'-'+str(month)+'-'+str(day) in file_name:
if (os.path.isfile(final_src)):
final_dest = dest + '\\ \\' + folder_name + '\\' + str(year) + '\\' + str(month) + '\\' + str(day)
shutil.copy(final_src, final_dest)
multithread.ADLUploader(lake, lpath=final_src, rpath=final_dest, nthreads=64, overwrite=True, buffersize=4194304, blocksize=4194304)
if ' ' in folder_path:
src_files = os.listdir(folder_path)
for file_name in src_files:
final_src = src + '\\' + folder_name + '\\' + file_name
for year in years:
for month in range(1, 12+1):
for day in range(1,31+1):
if len(str(month)) == 1 and len(str(day)) == 1:
if year+'-0'+str(month)+'-0'+str(day) in file_name:
if (os.path.isfile(final_src)):
final_dest = dest + '\\ \\' + folder_name + '\\' + str(year) + '\\' + str(month) + '\\' + str(day) + '\\' + file_name
shutil.copy(final_src, final_dest)
multithread.ADLUploader(lake, lpath=final_src, rpath=final_dest, nthreads=64, overwrite=True, buffersize=4194304, blocksize=4194304)
elif len(str(month)) == 1 and len(str(day)) == 2:
if year+'-0'+str(month)+'-'+str(day) in file_name:
if (os.path.isfile(final_src)):
final_dest = dest + '\\ \\' + folder_name + '\\' + str(year) + '\\' + str(month) + '\\' + str(day)
shutil.copy(final_src, final_dest)
multithread.ADLUploader(lake, lpath=final_src, rpath=final_dest, nthreads=64, overwrite=True, buffersize=4194304, blocksize=4194304)
else:
if year+'-'+str(month)+'-'+str(day) in file_name:
if (os.path.isfile(final_src)):
final_dest = dest + '\\ \\' + folder_name + '\\' + str(year) + '\\' + str(month) + '\\' + str(day)
shutil.copy(final_src, final_dest)
multithread.ADLUploader(lake, lpath=final_src, rpath=final_dest, nthreads=64, overwrite=True, buffersize=4194304, blocksize=4194304)
python azure-data-lake