自动创建目录Azure Data Lake Gen 2-PowerShell

Question

我想以这种格式在Azure Data Lake Gen2中创建目录。YYYY=2020(本年度(->MM=10(当前月份(，DD=28(当前日期(，并且我需要在Date文件夹中放置数据文件(data.csv(。即：/YYYY=2020/MM=10/DD=28/data.csv.

根文件夹目录中用于Data.csv的代码： 在@Jim Xu的帮助下

$username = "[e-mail]"
$password =ConvertTo-SecureString "" -AsPlainText -Force
$credential = New-Object System.Management.Automation.PsCredential($username,$password)

#Connect-AzureRmAccount -Credential $credential | out-null
Connect-AzAccount -Credential $credential
$dataFactoryName=""
$resourceGroupName=""
# get dataFactory triggers
$triggers=Get-AzDataFactoryV2Trigger -DataFactoryName $dataFactoryName  -ResourceGroupName $resourceGroupName
$datas=@()
foreach ($trigger in $triggers) {
# get the trigger run history
$today = Get-Date
$yesterday = $today.AddDays(-1)
$splat = @{ 
ResourceGroupName       = $trigger.ResourceGroupName
DataFactoryName         = $trigger.DataFactoryName
TriggerName             = $trigger.Name
TriggerRunStartedAfter  = $yesterday
TriggerRunStartedBefore = $today
}

$historys =Get-AzDataFactoryV2TriggerRun @splat
if($historys -ne $null){
# create date
foreach($history in $historys){
$obj =[PsCustomObject]@{
'TriggerRunTimestamp '     = $history.TriggerRunTimestamp
'ResourceGroupName '   =$history.ResourceGroupName
'DataFactoryName' =$history.DataFactoryName
'TriggerName '  = $history.TriggerName
'TriggerRunId'= $history.TriggerRunId
'TriggerType'=$history.TriggerType
'Status' =$history.Status
}
# add data to an array
$datas += $obj
}
} 


}
#  convert data to csv string
$contents =(($datas | ConvertTo-Csv -NoTypeInformation) -join [Environment]::NewLine)
# upload to Azure Data Lake Store Gen2
#1. Create a sas token
$accountName="testadls05"
$fileSystemName="test"
$filePath="data.csv"
$account = Get-AzStorageAccount -ResourceGroupName andywin7 -Name $accountName
$sas= New-AzStorageAccountSASToken -Service Blob  -ResourceType Service,Container,Object `
-Permission "racwdlup" -StartTime (Get-Date).AddMinutes(-10) `
-ExpiryTime (Get-Date).AddHours(2) -Context $account.Context
$baseUrl ="https://{0}.dfs.core.windows.net/{1}/{2}{3}" -f $accountName ,  $fileSystemName, $filePath, $sas
#2. Create file
$endpoint =$baseUrl +"&resource=file"
Invoke-RestMethod -Method Put -Uri $endpoint -Headers @{"Content-Length" = 0} -UseBasicParsing
#3 append data
$endpoint =$baseUrl +"&action=append&position=0"
Invoke-RestMethod -Method Patch -Uri $endpoint -Headers @{"Content-Length" = $contents.Length} -Body $contents -UseBasicParsing
#4 flush data
$endpoint =$baseUrl + ("&action=flush&position={0}" -f $contents.Length)
Invoke-RestMethod -Method Patch -Uri $endpoint -UseBasicParsing
#Check the result (get data)
Invoke-RestMethod -Method Get -Uri $baseUrl -UseBasicParsing

如果有人有建议，请发帖。感谢

Answer 1

这是我的解决方案：

$username = "[e-mail]"
$password = Get-Content D:Powershellnewpasswordspassword.txt | ConvertTo-SecureString -Key (Get-Content D:Powershellnewpasswordsaes.key)
$credential = New-Object System.Management.Automation.PsCredential($username,$password)
Connect-AzAccount -Credential $credential
# Input Variables
$dataFactoryName="dna-production-gen2"
$resourceGroupName="DataLake-Gen2"
# get dataFactory triggers
$triggers=Get-AzDataFactoryV2Trigger -DataFactoryName $dataFactoryName  -ResourceGroupName $resourceGroupName
$datas=@()
foreach ($trigger in $triggers) {
# get the trigger run history
$today = Get-Date
$yesterday = $today.AddDays(-1)
$splat = @{ 
ResourceGroupName       = $trigger.ResourceGroupName
DataFactoryName         = $trigger.DataFactoryName
TriggerName             = $trigger.Name
TriggerRunStartedAfter  = $yesterday
TriggerRunStartedBefore = $today
}

$historys =Get-AzDataFactoryV2TriggerRun @splat
if($historys -ne $null){
# create date
foreach($history in $historys){
$obj =[PsCustomObject]@{
'TriggerRunTimestamp '     = $history.TriggerRunTimestamp
'ResourceGroupName '   =$history.ResourceGroupName
'DataFactoryName' =$history.DataFactoryName
'TriggerName '  = $history.TriggerName
'TriggerRunId'= $history.TriggerRunId
'TriggerType'=$history.TriggerType
'Status' =$history.Status
}
# add data to an array
$datas += $obj
}
} 


}
#  convert data to csv string
$contents =(($datas | ConvertTo-Csv -NoTypeInformation) -join [Environment]::NewLine)
# upload to Azure Data Lake Store Gen2
#1. Create a sas token

$accountName="dna2020gen2"

# $path = New-Item -ItemType Directory -Path ".$((Get-Date).ToString('yyyy-MM-dd'))"
$YY = (Get-Date).year
$MM = (Get-Date).month
$DD = get-date –f dd

$fileSystemName="dev"
$filePath="triggers/YYYY=$YY/MM=$MM/DD=$DD/data.csv"
$account = Get-AzStorageAccount -ResourceGroupName 'DataLake-Gen2' -Name $accountName
$sas= New-AzStorageAccountSASToken -Service Blob  -ResourceType Service,Container,Object `
-Permission "racwdlup" -StartTime (Get-Date).AddMinutes(-10) `
-ExpiryTime (Get-Date).AddHours(2) -Context $account.Context
$baseUrl ="https://{0}.dfs.core.windows.net/{1}/{2}{3}" -f $accountName ,  $fileSystemName, $filePath, $sas
#2. Create file
$endpoint =$baseUrl +"&resource=file"
Invoke-RestMethod -Method Put -Uri $endpoint -Headers @{"Content-Length" = 0} -UseBasicParsing
#3 append data
$endpoint =$baseUrl +"&action=append&position=0"
Invoke-RestMethod -Method Patch -Uri $endpoint -Headers @{"Content-Length" = $contents.Length} -Body $contents -UseBasicParsing
#4 flush data
$endpoint =$baseUrl + ("&action=flush&position={0}" -f $contents.Length)
Invoke-RestMethod -Method Patch -Uri $endpoint -UseBasicParsing
#Check the result (get data)
Invoke-RestMethod -Method Get -Uri $baseUrl -UseBasicParsing