my net house

WAHEGURU….!

Go and Python

Lots of stuff for GO and Python: Important to know and read as well.
http://govspy.peterbe.com/

RUN Timescale after HUGE efforts

My Docker command:

docker run -d --name timescaledbworks -p 5000:5432 -e POSTGRES_PASSWORD=test -e POSTGRES_USER=test -e POSTGRES_DB=test timescale/timescaledb:latest-pg11

my GoLnag Source:

package main

import (
"database/sql"
_ "github.com/lib/pq"
"fmt"

)

func main() {
var err error
// test without port thing
connStr := "user=test dbname=test password=test host=0.0.0.0 sslmode=disable port=5000"
db, err := sql.Open("postgres", connStr)
if err != nil {
fmt.Println("could not connecet to Daatabase")
}
fmt.Println(db)
if err := db.Ping(); err != nil {
fmt.Println("unable to ping DB : " + err.Error())
return
}
//else{
//fmt.Println("able to connecet to db")
//}

//db.Query("CREATE TABLE conditions(time TIMESTAMP NOT NULL,location TEXT NOT NULL,temperature DOUBLE PRECISION NULL,humidity DOUBLE PRECISION NULL);")
//db.Query("CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE;")
//db.Query("CREATE TABLE hello")
//db.Query("SELECT create_hypertable('conditions','time','location',4);")
}

Hampering TimeScale-DB over Cup of Tea

Let’s hamper TimeScale DB over Tea and find out how much Day it can save:

Hardware Requirements:

OS: Ubuntu 16.04

CPU/RAM – Third generation i3 / 10 GB

Hyper Threading: Enabled

What is TimeScale?

If this is your First Try with Postgress, make sure You know How to reset Password for Postgress.

Following code we are able to do all the CRUD operations using SQL Language only.


--creating a simpe table

CREATE TABLE conditions (
time TIMESTAMP NOT NULL,
location TEXT NOT NULL,
temperature DOUBLE PRECISION NULL,
humidity DOUBLE PRECISION NULL);

-- creating a hypertable. now it wil be partioned with location

SELECT create_hypertable('conditions','time','location',4);

INSERT INTO conditions(time, location, temperature, humidity)
VALUES (NOW(), 'office', 70.0, 50.0);

SELECT * FROM conditions ORDER BY time DESC LIMIT 100;

-- upda<span id="mce_SELREST_start" style="overflow:hidden;line-height:0;"></span>ting data on Hyper tables

UPDATE conditions SET temperature = 20000, humidity = 50000
WHERE time = '2019-02-17 12:45:02.189422' AND location = 'office';

SELECT * FROM conditions ORDER BY temperature DESC LIMIT 100;

-- upserting data

-- create it with unique IDs

CREATE TABLE conditions_unique (
time TIMESTAMPTZ NOT NULL,
location TEXT NOT NULL,
temperature DOUBLE PRECISION NULL,
humidity DOUBLE PRECISION NULL,
UNIQUE (time, location)
);
INSERT INTO conditions_unique(time, location, temperature, humidity)
VALUES (NOW(), 'office', 70.0, 50.0);
INSERT INTO conditions_unique
VALUES ('2019-02-17 12:56:05.080302+05:30', 'office', 898900000000.1, 9879678000000000.0)
ON CONFLICT DO NOTHING;

SELECT * FROM conditions_unique ORDER BY temperature DESC LIMIT 100;

It is as easy to play with bit complex quiries as one is doing Python. 😉


-- Before going with this you have to download and setup NYC data
-- Folow this link https://docs.timescale.com/v1.2/tutorials/tutorial-hello-nyc

\c nyc_data

SELECT date_trunc('day', pickup_datetime) as day, avg(fare_amount)
FROM rides
WHERE passenger_count > 1 AND pickup_datetime < '2016-01-08'
GROUP BY day ORDER BY day;

SELECT date_trunc('day', pickup_datetime) as day, COUNT(*) FROM rides
GROUP BY day ORDER BY day
LIMIT 5;

SELECT time_bucket('5 minute', pickup_datetime) AS five_min, count(*)
FROM rides
WHERE pickup_datetime < '2016-01-01 02:00'
GROUP BY five_min ORDER BY five_min;

SELECT rate_code, COUNT(vendor_id) as num_trips FROM rides
WHERE pickup_datetime < '2016-01-08'
GROUP BY rate_code ORDER BY rate_code;

-- Join rides with rates to get more information on rate_code
SELECT rates.description, COUNT(vendor_id) as num_trips FROM rides
JOIN rates on rides.rate_code = rates.rate_code
WHERE pickup_datetime < '2016-01-08'
GROUP BY rates.description ORDER BY rates.description;

now let's Play with more complex stuff but see how far we can go?

  1. How large should I configure my intervals for time partitioning?
  2. Should I use space partitioning, and how many space partitions should I use?

Look for Best Practices:> https://docs.timescale.com/v1.2/using-timescaledb/hypertables


-- ============================================================
-- Structure
-- ============================================================

CREATE DATABASE test;

\c test;

CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE;

CREATE TABLE collfact (
time TIMESTAMP NOT NULL,
pointid BIGINT NOT NULL,
value DOUBLE PRECISION NULL,
mesureid BIGINT NULL,
version INT NULL,
insertiondate TIMESTAMP NULL,
iscurrent CHAR(1)
);

SELECT create_hypertable('collfact', 'time', 'pointid', 16, chunk_time_interval => interval '1 day');
create index on collfact(time, pointid);

-- ============================================================
-- Generate test data, ~ 1 year
-- ============================================================

-- meter location, or point
create table point(
pointid BIGINT NOT NULL PRIMARY KEY,
pointnm VARCHAR(20) NOT NULL
);

CREATE SEQUENCE pointid_seq START 1;

CREATE OR REPLACE FUNCTION genPoints(in a_no_of_records integer) RETURNS integer AS $$
DECLARE
v_counter integer;
vhash bigint;
BEGIN
v_counter := 1;
RAISE NOTICE 'No of records to insert : %', a_no_of_records;
WHILE (v_counter <= a_no_of_records) LOOP
IF( v_counter % 10000 =0) THEN
RAISE NOTICE 'Counter here is %', v_counter;
END IF;
vhash := nextval('pointid_seq');

--insert into partiton table
INSERT INTO point(pointid, pointnm)
VALUES (vhash, 'Point num.' || vhash);
v_counter := v_counter + 1;
END LOOP;
RETURN 0;
END;

$$ LANGUAGE plpgsql;

-- Meter creation
select * from genPoints(10000);

create table time (
time TIMESTAMP NOT NULL PRIMARY KEY
);

-- Year 2018, step 15'
INSERT INTO time(time)
SELECT dd
FROM generate_series
(timestamp '2018-01-01 00:00:00'
, timestamp '2018-12-31 23:45:00'
, '15 minutes'::interval) dd ;

-- mesure type table; just 4 here
CREATE TABLE mesure (
mesureid BIGINT NOT NULL PRIMARY KEY,
mesurenm VARCHAR(20)
);

INSERT INTO MESURE (mesureid, mesurenm) VALUES (1, 'P+');
INSERT INTO MESURE (mesureid, mesurenm) VALUES (2, 'P-');
INSERT INTO MESURE (mesureid, mesurenm) VALUES (3, 'Q+');
INSERT INTO MESURE (mesureid, mesurenm) VALUES (4, 'Q-');

\timing on

-- INSERT for 1000 meters, 2 mesure types
insert into collfact (time, pointid, value, mesureid, version, insertiondate, iscurrent)
select time, pointid, trunc(random() * 99999), 1 as mesureid, 1, now(), '1'
from
(select * from time) as a,
(select * from point limit 1000) as b
UNION ALL
select time, pointid, trunc(random() * 99999), 2 as mesureid, 1, now(), '1'
from
(select * from time) as a,
(select * from point limit 1000) as b;

-- INSERT for 1000 meters, 2 mesure types
insert into collfact (time, pointid, value, mesureid, version, insertiondate, iscurrent)
select time, pointid, trunc(random() * 99999), 3 as mesureid, 1, now(), '1'
from
(select * from time) as a,
(select * from point limit 1000) as b;

I am running out of Shared memory after 15-20 mintues by using above code.

Yo can use code from this Git repo as well:

https://github.com/arshpreetsingh/timeScale-analysis

My findings:

1. You need to spend time over optmization of DB Setup like how much memory you want to share and should look for how to Optimize PGsql on your System.

2. You must carefully understand your data and how much Chunck-Size you want to define.

3. You don’t have to think much while writing complex Quiries because writing complex quiries is straight forward as well.

4. There is very good CRUD support.

5. You can perform Data-Analysis, Visualization and Machine-Learning very easily because community is supporting it on the FLY.

6. As it is extension of PG-SQL so stability can be defined.

Some Gitty GITs

When you don’t want to type username an password each time or when you are not allowed to do so as well.

Your repo should look like this:

git@github.com:USERNAME/REPOSITORY.git

you can see config using:


nano .git/config

SSH key can be generated like:

ssh-keygen -t rsa -b 4096 -C “arshpreet.singh@myemail.com”

 

More about Git and adding ssh key can be found here:
https://help.github.com/articles/connecting-to-github-with-ssh/

Click – Command Line Interface Creation Kit , OH yeahh Python

This is an Easy to do things in python so we can run scripts using command line Arguments for us as well.


import click 

@click.command()
@click.option('--count',default = 1, )
@click.option('--name',prompt = "What you are supposed to do here?")
def test_click(count,name):
    for i in range(count):
        click.echo(str(name)) 
'''
How to run this script:

python test_click.py --name "Arsh" --count 4

'''

Learn Ruby in Next 30 Minutes

Right Now time is: 5:00 , Let’s complete this by 5:30

# This is all about ROR on print statements :

puts ‘Hello, World!’
puts ‘Hello world’+’how are you’

# how to format a string in ruby

puts “The number is %d” % [20]

# This is al about Variables and Multiline comments:

=begin
in order,
a string,
a boolean,
and a number
=end
aString = “I’m a string!”
aBoolean = true
aNumber = 42
afloatMaybe = 10.0
afraction_could_be = 25/5
puts “this is a flaot %d” % [afraction_could_be]

# This is all about “UGLY” Math we are doing with Ruby 😛 muhahahhaha

numberOne = 6
numberTwo = 8
numberThree = 5
numberFour = 12
numberFive = 36

testOne = numberTwo*numberThree #fill in the blank to make this 40
testTwo = numberFive-6*numberOne #fill in the blank to make this 0
testThree = numberFour-numberThree #fill in the blank to make this 7
if testOne == 40 && testTwo == 0 && testThree == 7
puts true
end

# Lots about string Manipulation and stuff as well.

myString = “Hi! I am code!” #In the next line, use methods to change it.
myNewString = [myString.upcase,myString.downcase,myString.reverse]
puts myNewString

# so much more about Arrays and processing:

myArray = [“Not me!”, “Not me!”, “Me!”, “Not me!”]
arr = [11,22,33,44,55]
arr.each_with_index {|val, index| puts “#{index >= 0 && index “this is my hash value 1″,”Mum”=>”this is my Hash value2”}
puts myFirstHash[“Dad”]

# This condition is all about If ELse and bla bla bla

language = “No Language”
if language === “English”
puts “Hello!”
elsif language === “Spanish”
puts “Hola!”
else
puts “this is the condition where all is well”
end

# unless, until and bla bla bla
#Make your loop below!
a = 2
b = 4
unless a>b
puts “omg!!!”
end

until a>b
puts “oye-hoyeeee”
end

# this is all about methods we need to know:

def methodname
puts “how to execute methods”
end

methodname()

For classes start from here:
https://www.tutorialspoint.com/ruby/ruby_classes.htm

OOPS it is still on. 😦

Python Send SMS all over the things

This is Python Script that you can use to send Messages on any Phone without leaving trace of any number.


# python script for sending message update 

import time 
from time import sleep 
from sinchsms import SinchSMS 

# function for sending SMS 
def sendSMS(): 

	# enter all the details 
	# get app_key and app_secret by registering 
	# a app on sinchSMS 
	number = '+919915959387'
	app_key = '29d0e101-e619-4226-823a-c39c4b99b5ac'
	app_secret = '1WkUjJ3RWEy1P9h9mw/2sQ=='

	# enter the message to be sent 
	message = "hello"

	client = SinchSMS(app_key, app_secret) 
	print("Sending '%s' to %s" % (message, number)) 

	response = client.send_message(number, message) 
	message_id = response['messageId'] 
	response = client.check_status(message_id) 

	# keep trying unless the status retured is Successful 
	while response['status'] != 'Successful': 
		print(response['status']) 
		time.sleep(1) 
		response = client.check_status(message_id) 

	print(response['status']) 

if __name__ == "__main__": 
	sendSMS()
 

Logging and importance of it with examples

Python and Flask supports wide range of logging as well. Either it’s warning, error or just a logger you can ago through all of those in very specific instance of time.


Logging is important of the Maintainability of the application.

Now logging is something like you need to go for when you see or feel that your web app needs lots of “Watching as well!”

Here is simple Example in Flask:

import logging
from logging.handlers import RotatingFileHandler

from flask import Flask

app = Flask(__name__)

@app.route('/')
def foo():
app.logger.warning('A warning occurred (%d apples)', 42)
app.logger.error('An error occurred')
app.logger.info('Info')
return "foo"

if __name__ == '__main__':
handler = RotatingFileHandler('foo.log', maxBytes=10000, backupCount=1)
handler.setLevel(logging.INFO)
app.logger.addHandler(handler)
app.run()

[/code ]

for more detailed view on logging and system you can go for the following link as well. :

It’s very much explanatory: https://gist.github.com/mariocj89/73824162a3e35d50db8e758a42e39aab

install,setup and Play with Mongo Python

This is the good way to do stuff with Mongo, Python and other.

https://hevodata.com/blog/install-mongodb-on-ubuntu/

 

Deploy Micro-service in Seconds Using Falcon

Falcon framework is great when you have to build micro-services in Python, All we need to do is create a Class that would provide the specific Purpose to Establish one Micro-service as Route and add the route to the API would be able to server as deploy multiple modes inside the Falcon Architecture as well.


# sample.py

import falcon

class QuoteResource:
def on_get(self, req, resp):
"""Handles GET requests"""
quote = {
'quote': (
"I've always been more interested in "
"the future than in the past."
),
'author': 'Grace Hopper'
}

resp.media = quote
class HelpObject:
def on_get(self, req, resp):
"""Handles GET requests"""
quote = {
'quote': (
"I've always been more interested in "
"the future than in the past."
),
'author': 'Arshpreet Singh'
}

resp.media = quote

api = falcon.API()
api.add_route('/quote', QuoteResource())
api.add_route('/quote2', HelpObject())
<span id="mce_SELREST_start" style="overflow:hidden;line-height:0;">&#65279;</span>
%d bloggers like this: