To protect your data, the CISO officer has suggested users to enable 2FA as soon as possible.
Currently 2.7% of users enabled 2FA.

Commit dfe69c52 authored by Liam Hayes's avatar Liam Hayes
Browse files

initial commit. website works, but many helper scripts are missing

parents
# general
__pycache__
/db.sqlite3
/media
/windatlas.egg-info
# results files
/scripts/experiments/results
\ No newline at end of file
__version__ = '0.1.0'
from .drive_reader import DriveReader
from .s3_reader import S3Reader
\ No newline at end of file
import numpy as np, pandas as pd, os
from datetime import datetime, timedelta
from .n320 import N320
NUM_N320_PTS = 542080
class DriveReader(object):
'''class for reading downloaded ERA5 data that has been processed into
.npz files'''
def __init__(self, data_dir, config):
self.data_dir = data_dir
self.config = config
n320_dir = os.path.join(os.path.dirname(__file__), 'data/n320/')
self.n320 = N320(n320_dir)
def __get_fname(self, path, i_n320):
'''path to npz file that stores the particular n320 grid point data'''
num_files = self.config[path][4]
num_n320_pts_per_file = NUM_N320_PTS // num_files
i_file = i_n320 // num_n320_pts_per_file
i_n320_start = i_file * num_n320_pts_per_file
i_n330_end = i_n320_start + num_n320_pts_per_file - 1
return path + f'{i_n320_start}_{i_n330_end}.npz'
def __df_format(date_from, date_to, **kwargs):
time = np.arange(date_from, date_to, timedelta(hours=1))
df = pd.DataFrame(kwargs, index=time)
df.index.name = 'datetime'
return df
def wind_speed_at_height(self, date_from, date_to, height, lat, lon, interpolated=True, format=None):
wind131 = self.wind_speed_at_level(date_from, date_to, 131, lat, lon, interpolated=interpolated)
wind133 = self.wind_speed_at_level(date_from, date_to, 133, lat, lon, interpolated=interpolated)
wind135 = self.wind_speed_at_level(date_from, date_to, 135, lat, lon, interpolated=interpolated)
x = np.array([169.5, 106.54, 53.92])
y = np.array([wind131, wind133, wind135], dtype=np.float32)
a, b = np.polynomial.polynomial.polyfit(np.log(x), y, 1)
data = a + b * np.log(height)
if format == 'dataframe':
data = DriveReader.__df_format(date_from, date_to, wind_speed=data)
return data
def wind_speed_at_level(self, date_from, date_to, level, lat, lon, interpolated=True, format=None):
u = self.read_data(date_from, date_to, level, 'u', lat, lon, interpolated)
v = self.read_data(date_from, date_to, level, 'v', lat, lon, interpolated)
data = np.sqrt(u**2 + v**2)
if format == 'dataframe':
data = DriveReader.__df_format(date_from, date_to, wind_speed=data)
return data
def read_data(self, date_from, date_to, level, param, lat=None, lon=None, interpolated=False, n320_index=None):
'''
from_date, to_date: datetime objects, inclusive and exclusive
level: int, ERA5 model level
param: str, 'u' for eastward wind, 'v' for northward wind, 't' for temperature
lat, lon: floats, WGS84
interpolated: bool, False means use the closest n320 grid point
'''
# date_from and to_date represent the requested period
# date1 and date2 represent the period STORED in a particular download directory
# dateA and dateB represent the period NEEDED from a particular download directory
# all of them are inclusive, exclusive respectively
periods = {} # similar format as config (without level and param)
temp_date = date_from
while temp_date < date_to:
for path,(date1,date2,lvl,params,num_files) in self.config.items():
if level == lvl and param in params:
if temp_date >= date1 and temp_date < date2:
periods[path] = (temp_date, min(date2, date_to))
temp_date = date2
break
else:
raise Exception('[!] no data avaliable (date={}, param={})'.format(
temp_date, param
))
hour = timedelta(hours=1)
data = np.zeros((date_to - date_from) // hour) ### dtype='float16' to keep small
for path,(dateA,dateB) in periods.items():
date1 = self.config[path][0] # start of STORED downloaded
nhrs = (dateB - dateA) // hour
i_src = (dateA - date1) // hour
i_dst = (dateA - date_from) // hour
if n320_index:
weights = {n320_index: 1.0}
elif interpolated:
weights = self.n320.latlon_to_n320_interpolated(lat, lon)
else:
weights = {self.n320.latlon_to_n320(lat, lon): 1.0}
for i_n320, weight in weights.items():
fname = self.data_dir + self.__get_fname(path, i_n320)
array_name = '{}_{}'.format(i_n320, param.lower())
array = np.load(fname)[array_name] * weight
data[i_dst:(i_dst+nhrs)] += array[i_src:(i_src+nhrs)]
return data
import pyproj, math, numpy as np
class N320(object):
'''Class for helping convert between latitudes / longitudes and
grid points on an N320 reduced Gaussian grid.
See https://confluence.ecmwf.int/display/UDOC/N320'''
def __init__(self, n320_dir):
# consts
self.NUM_N320_PTS = 542080
# input arrays
self.lats = np.load(n320_dir + 'latitudes.npy') # the latitudes of the N320 grid
self.lons = np.load(n320_dir + 'pts_per_latitude.npy') # num of evenly spaced longitudes on each of those latitudes
# useful arrays:
self.i_lons_end = np.cumsum(self.lons) # last index (exclusive) of the ith latitude
self.i_lons_start = np.zeros(self.lons.shape, dtype='int')
self.i_lons_start[1:] = self.i_lons_end[:-1] # first index (inclusive) of the ith latitude
# for distances
self.geod = pyproj.Geod(ellps='WGS84')
def latlon_to_n320(self, lat, lon):
'''convert a lat + lon the closest n320 index'''
i_lat = np.argmin(np.abs(self.lats - lat)) # index into lats of closest latitude
i_lon_start = self.i_lons_start[i_lat] # index into flat n320 array to get to start of said latitude
num_divs = self.lons[i_lat] # split up 360 degs of longitude by number of divisions
i_lon = int(round(lon / 360.0 * num_divs, 0)) % num_divs # index of closest one
# print(num_divs, i_lon)
return i_lon_start + i_lon
def n320_to_latlon(self, i_n320):
'''convert a n320 index to lat + lon'''
i_lat = np.sum(i_n320 >= self.i_lons_end)
lat = self.lats[i_lat]
i_lon_start = self.i_lons_start[i_lat]
i_lon = i_n320 - i_lon_start
num_divs = self.lons[i_lat]
lon = (i_lon * 360.0 / num_divs) % 360
return lat, lon
# lat is -90 to 90, lon is 0 to 360
def __dist(self, lat1, lon1, lat2, lon2):
'''distance in m between two points'''
dist = self.geod.inv(lon1, lat1, lon2, lat2)[2]
return max(dist, 0.0000000001) # to avoid divide by 0 errors
def latlon_to_n320_interpolated(self, lat, lon):
'''find the 4 closest N320 grid points in the NW, NE, SW, SE directions
assign each a weight to use for interpolation (inverse distance to original point)
this is not proper bilinear quadrilateral interpolation, but it's pretty good'''
# bound lat within range
if lat > self.lats[0]:
lat = self.lats[0]
elif lat < self.lats[-1]:
lat = self.lats[-1]
# get the two closest latitudes
i_lat1, i_lat2 = np.argpartition(np.abs(self.lats - lat), 2)[:2]
lat1 = self.lats[i_lat1]
i_lon_start1 = self.i_lons_start[i_lat1]
lat2 = self.lats[i_lat2]
i_lon_start2 = self.i_lons_start[i_lat2]
# get two closest points on one latitude
num_divs1 = self.lons[i_lat1]
i_lon1A = math.floor(lon / 360.0 * num_divs1) % num_divs1
i_lon1B = math.ceil(lon / 360.0 * num_divs1) % num_divs1
if i_lon1A == i_lon1B:
i_lon1B = (i_lon1B + 1) % num_divs1
lon1A = i_lon1A * 360.0 / num_divs1
lon1B = i_lon1B * 360.0 / num_divs1
# get two closest points on other latitude
num_divs2 = self.lons[i_lat2]
i_lon2A = math.floor(lon / 360.0 * num_divs2) % num_divs2
i_lon2B = math.ceil(lon / 360.0 * num_divs2) % num_divs2
if i_lon2A == i_lon2B:
i_lon2B = (i_lon2B + 1) % num_divs2
lon2A = i_lon2A * 360.0 / num_divs2
lon2B = i_lon2B * 360.0 / num_divs2
results = {
i_lon_start1 + i_lon1A: 1 / self.__dist(lat, lon, lat1, lon1A),
i_lon_start1 + i_lon1B: 1 / self.__dist(lat, lon, lat1, lon1B),
i_lon_start2 + i_lon2A: 1 / self.__dist(lat, lon, lat2, lon2A),
i_lon_start2 + i_lon2B: 1 / self.__dist(lat, lon, lat2, lon2B),
}
scale = sum(results.values())
results = {k: v/scale for k,v in results.items()}
return results
import numpy as np, pandas as pd
import boto3, os, io
from datetime import datetime, timedelta
from .n320 import N320
class S3Reader(object):
'''class for reading ERA5 data that has been stored in a Amazon S3 bucket'''
def __init__(self, data_dir, config, dir_type='s3'):
assert(dir_type == 's3') # local storage not implemented
self.data_dir = data_dir
self.dir_type = dir_type
self.config = config
n320_dir = os.path.join(os.path.dirname(__file__), 'data/n320/')
self.n320 = N320(n320_dir)
def __load_array(self, path, i_n320):
'''path to npz file that stores the particular n320 grid point data'''
key = f'{path}{i_n320}.npy'
try:
npy = io.BytesIO(self.data_dir.Object(key).get()['Body'].read())
except:
raise Exception('Data does not exist for this location')
return np.load(npy)
def __df_format(date_from, date_to, **kwargs):
time = np.arange(date_from, date_to, timedelta(hours=1))
df = pd.DataFrame(kwargs, index=time)
df.index.name = 'datetime'
return df
def __read_data(self, date_from, date_to, height, i_n320):
# this function does the vertical interpolation
# date_from and date_to represent the REQUESTED period
# date1 and date2 represent the period STORED in a particular directory
# dateA and dateB represent the period NEEDED from a particular directory
# all of them are inclusive, exclusive respectively
periods = {} # similar format as config
temp_date = date_from
while temp_date < date_to:
for path, (date1,date2) in self.config.items():
if temp_date >= date1 and temp_date < date2:
periods[path] = (temp_date, min(date2, date_to))
temp_date = date2
break
else:
raise Exception(f'[!] no data avaliable (date={temp_date})')
hour = timedelta(hours=1)
a = np.full((date_to - date_from) // hour, 0.0, dtype='float16')
b = np.full((date_to - date_from) // hour, 0.0, dtype='float16')
for path, (dateA,dateB) in periods.items():
date1 = self.config[path][0] # start of STORED downloaded
nhrs = (dateB - dateA) // hour
i_src = (dateA - date1) // hour
i_dst = (dateA - date_from) // hour
array_a = self.__load_array(path + 'a/', i_n320)
a[i_dst:(i_dst+nhrs)] = array_a[i_src:(i_src+nhrs)]
array_b = self.__load_array(path + 'b/', i_n320)
b[i_dst:(i_dst+nhrs)] = array_b[i_src:(i_src+nhrs)]
return a + b * np.log(height)
def wind_speed(self, date_from, date_to, height, location):
# location can be integer (n320 index) or 2-tuple (lat,lon)
# height is in meters
# this function does horizontal interpolation
if isinstance(location, int):
i_n320 = location # location is n320 index (no interpolation)
return self.__read_data(date_from, date_to, height, i_n320)
elif isinstance(location, tuple):
lat, lon = location # location is lat,lon (use inverse distance interpolation)
weights = self.n320.latlon_to_n320_interpolated(lat, lon)
hour = timedelta(hours=1)
data = np.zeros((date_to - date_from) // hour, dtype='float16')
for i_n320, weight in weights.items():
data += self.__read_data(date_from, date_to, height, i_n320)*weight
return data
else:
raise Exception('[!] location must be int or tuple')
from django.contrib import admin
# Register your models here.
from django.apps import AppConfig
class FarmsConfig(AppConfig):
name = 'farms'
from django.views.generic import ListView, TemplateView
from django.conf import settings
import os
class MyListView(ListView):
'''A base class for ListViews that filters objects by
a provided set of GET parameters'''
def get_queryset(self):
objs = self.model.objects
if hasattr(self, 'filter_params'):
filters = {x: self.request.GET[x] for x in self.filter_params if x in self.request.GET}
objs = objs.filter(**filters)
if hasattr(self, 'order_param'):
objs = objs.order_by(self.order_param)
return objs
class MyMarkdownView(TemplateView):
'''A base class that passes the contents of the file "self.markdown_name"
to be rendered as markdown in "self.template_name"'''
template_name = 'farms/markdown.html'
def get_context_data(self, **kwargs):
context = super().get_context_data(**kwargs)
fname = os.path.join(settings.BASE_DIR, self.markdown_name)
with open(fname, 'r') as f:
context['markdown'] = f.read()
return context
\ No newline at end of file
from django.forms import ModelForm, Form
from django import forms
from crispy_forms.helper import FormHelper
from crispy_forms.layout import Layout, Submit, Row, Column
from .models import GwaStat, PowerCurve
from datetime import datetime, date
class GwaStatForm(ModelForm):
class Meta:
model = GwaStat
fields = ['name', 'lat', 'lon', 'roughness', 'height', 'gwc_file', 'plots_file']
class DateInput(forms.DateInput):
input_type = 'date'
class APIForm(Form):
lat = forms.FloatField(label='Latitude', initial=50.0)
lon = forms.FloatField(label='Longitude', initial=0.0)
height = forms.IntegerField(label='Hub height (m)', initial=100, min_value=0, max_value=500)
custom_date_range = forms.BooleanField(initial=False, required=False)
year = forms.IntegerField(
label='Year',
initial=2019,
widget=forms.Select(choices=[(x,x) for x in range(1980,2020)]),
)
date_from = forms.DateField(widget=DateInput, initial=date(2010, 1, 1))
date_to = forms.DateField(widget=DateInput, initial=date(2019, 12, 31))
period = forms.ChoiceField(
choices=[('hour', 'hourly'), ('day', 'daily'), ('month', 'monthly')],
widget=forms.RadioSelect,
initial='hour',
required=False,
)
calculate_capacity_factors = forms.BooleanField(initial=True, required=False)
turbine = forms.ModelChoiceField(
label='Wind turbine',
queryset=PowerCurve.objects.all().order_by('name'),
initial=PowerCurve.objects.get(name='Vestas V126-3450'),
required=False,
)
def __init__(self, *args, **kwargs):
super(APIForm, self).__init__(*args, **kwargs)
self.fields['calculate_capacity_factors'].widget.attrs['onclick'] = "javascript:updateForm();"
self.fields['custom_date_range'].widget.attrs['onclick'] = "javascript:updateForm();"
self.helper = FormHelper()
self.helper.layout = Layout(
Row(
Column('lat', css_class='form-group col-md-4 mb-0'),
Column('lon', css_class='form-group col-md-4 mb-0'),
Column('height', css_class='form-group col-md-4 mb-0'),
css_class='form-row'
),
'custom_date_range',
Row(
Column('year', css_class='form-group col-md-4 mb-0'),
id='row_id_year_selector',
css_class='form-row'
),
Row(
Column('date_from', css_class='form-group col-md-4 mb-0'),
Column('date_to', css_class='form-group col-md-4 mb-0'),
id='row_id_custom_date_selector',
css_class='form-row',
),
'calculate_capacity_factors',
'turbine',
'period',
Submit('download', 'Download Data'),
Submit('api-request', 'View API Request'),
)
def clean_date_to(value):
date_to = datetime.strptime(value.data['date_to'], '%Y-%m-%d').date()
if 'custom_date_range' in value.data:
if date_to < date(1980,1,1) or date_to >= date(2020,1,1):
raise forms.ValidationError('Must be in range 1980 to 2019')
date_from = datetime.strptime(value.data['date_from'], '%Y-%m-%d').date()
if date_to < date_from:
raise forms.ValidationError('Must be greater than "Date from"')
return date_to
def clean_date_from(value):
date_from = datetime.strptime(value.data['date_from'], '%Y-%m-%d').date()
if 'custom_date_range' in value.data:
if date_from < date(1980,1,1) or date_from >= date(2020,1,1):
raise forms.ValidationError('Must be in range 1980 to 2019')
return date_from
def clean_lat(value):
lat = float(value.data['lat'])
if lat < -90 or lat > 90:
raise forms.ValidationError('Must be in range -90 to 90')
return lat
def clean_lon(value):
lon = float(value.data['lon'])
if lon < -180 or lon > 180:
raise forms.ValidationError('Must be in range -180 to 180')
return lon
def clean_height(value):
height = int(value.data['height'])
if height < 0 or height > 500:
raise forms.ValidationError('Must be in range 0 to 500')
return height
\ No newline at end of file
# Generated by Django 3.1.4 on 2020-12-03 06:37
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
dependencies = [
]
operations = [
migrations.CreateModel(
name='GwaStat',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(max_length=100, unique=True)),
('lat', models.FloatField()),
('lon', models.FloatField()),
('gwc_file', models.FileField(upload_to='GwaStat/gwc_file')),
('plots_file', models.FileField(upload_to='GwaStat/plot_file')),
('roughness', models.FloatField()),
('height', models.FloatField()),
],
),
]
# Generated by Django 3.1.4 on 2020-12-03 06:38
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('farms', '0001_initial'),
]
operations = [
migrations.CreateModel(
name='Farm',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(max_length=100, unique=True)),
('lat', models.FloatField()),
('lon', models.FloatField()),
('height', models.FloatField()),
('num_turbines', models.IntegerField()),
('capacity', models.FloatField()),
('buildtype', models.CharField(max_length=100)),
('country', models.CharField(max_length=100)),
('date_start', models.DateTimeField()),
('date_end', models.DateTimeField()),
('turbine_model', models.CharField(max_length=100)),
],
),
migrations.CreateModel(
name='PowerCurve',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(max_length=100, unique=True)),
('power', models.FloatField()),
('diameter', models.FloatField()),
('source', models.CharField(max_length=100)),
('data', models.FileField(upload_to='PowerCurve/data')),
],
),
migrations.CreateModel(
name='SeriesEra5Wind',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('lat', models.FloatField()),
('lon', models.FloatField()),
('height', models.FloatField()),
('date_start', models.DateTimeField()),
('date_end', models.DateTimeField()),
('data', models.FileField(upload_to='SeriesEra5Wind/data')),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='SeriesNinjaWind',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('lat', models.FloatField()),
('lon', models.FloatField()),
('height', models.FloatField()),
('date_start', models.DateTimeField()),
('date_end', models.DateTimeField()),
('data', models.FileField(upload_to='SeriesNinjaWind/data')),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='Trace',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(max_length=100, unique=True)),
('uid', models.CharField(max_length=100, unique=True)),
('source', models.CharField(max_length=100)),
('date_start', models.DateTimeField()),
('date_end', models.DateTimeField()),