def ensure_watchers(owner, repo)
      currepo = ensure_repo(owner, repo)

      if currepo.nil?
        warn "Could not find repo #{owner}/#{repo} for retrieving watchers"
        return
      end

      watchers = db.from(:watchers, :users).\
          where(Sequel.qualify('watchers', 'user_id') => Sequel.qualify('users', 'id')).\
          where(Sequel.qualify('watchers', 'repo_id') => currepo[:id]).select(:login).all

      retrieve_watchers(owner, repo).reduce([]) do |acc, x|
        if watchers.find { |y|
          y[:login] == x['login']
        }.nil?
          acc << x
        else
          acc
        end
      end.map { |x| save{ensure_watcher(owner, repo, x['login']) }}.select{|x| !x.nil?}
    end

Make sure that all watchers exist for a repository


def ensure_pull_requests(owner, repo, refresh = false)
      currepo = ensure_repo(owner, repo)
      if currepo.nil?
        warn "Could not find repo #{owner}/#{repo} for retrieving pull requests"
        return
      end

      raw_pull_reqs = if refresh
                        retrieve_pull_requests(owner, repo, refresh = true)
                      else
                        pull_reqs = db[:pull_requests].filter(:base_repo_id => currepo[:id]).all
                        retrieve_pull_requests(owner, repo).reduce([]) do |acc, x|
                          if pull_reqs.find { |y| y[:pullreq_id] == x['number'] }.nil?
                            acc << x
                          else
                            acc
                          end
                        end
                      end

      raw_pull_reqs.map { |x| save { ensure_pull_request(owner, repo, x['number']) } }.select { |x| !x.nil? }
    end


def ensure_pull_request_history(id, ts, act, actor)
      user = unless actor.nil?
               ensure_user(actor, false, false)
             end
      pull_req_history = db[:pull_request_history]

      entry =  if ['opened', 'merged'].include? act
                  pull_req_history.first(:pull_request_id => id,
                                         :action => act)
               else
                 pull_req_history.first(:pull_request_id => id,
                                        :created_at => (ts - 3)..(ts + 3),
                                        :action => act)
               end

      if entry.nil?
        pull_req_history.insert(:pull_request_id => id,
                                :created_at => ts,
                                :action => act,
                                :actor_id => unless user.nil? then user[:id] end)
        info "Added pullreq_event (#{id}) -> (#{act}) by (#{actor}) timestamp #{ts}"
      else
        debug "Pull request (#{id}) event (#{act}) by (#{actor}) timestamp #{ts} exists"
        if entry[:actor_id].nil? and not user.nil?
          pull_req_history.where(:pull_request_id => id,
                               :created_at => (ts - 3)..(ts + 3),
                               :action => act)\
                          .update(:actor_id => user[:id])
          info "Updated pull request (#{id}) event (#{act}) timestamp #{ts}, actor -> #{user[:login]}"
        end
      end
    end


def pr_is_intra_branch(req)
      return false unless pr_has_head_repo(req)

      if req['head']['repo']['owner']['login'] ==
          req['base']['repo']['owner']['login'] and
          req['head']['repo']['full_name'] == req['base']['repo']['full_name']
        true
      else
        false
      end
    end

def pr_is_intra_branch ( req ) return false unless pr_has_head_repo ( req ) if req [ 'head' ] [ 'repo' ] [ 'owner' ] [ 'login' ] == req [ 'base' ] [ 'repo' ] [ 'owner' ] [ 'login' ] and req [ 'head' ] [ 'repo' ] [ 'full_name' ] == req [ 'base' ] [ 'repo' ] [ 'full_name' ] true else false end end

Checks whether a pull request concerns two branches of the same
 repository


def ensure_forks(owner, repo)
      currepo = ensure_repo(owner, repo)

      if currepo.nil?
        warn "Could not find repo #{owner}/#{repo} for retrieving forks"
        return
      end

      existing_forks = db.from(:projects, :users).\
          where(Sequel.qualify('users', 'id') => Sequel.qualify('projects', 'owner_id')). \
          where(Sequel.qualify('projects', 'forked_from') => currepo[:id]).\
          select(Sequel.qualify('projects', 'name'), :login).all

      retrieve_forks(owner, repo).reduce([]) do |acc, x|
        if existing_forks.find do |y|
          forked_repo_owner = x['url'].split(/\//)[4]
          forked_repo_name = x['url'].split(/\//)[5]
          y[:login] == forked_repo_owner && y[:name] == forked_repo_name
        end.nil?
          acc << x
        else
          acc
        end
      end.map { |x| save{ensure_fork(owner, repo, x['id']) }}.select{|x| !x.nil?}
    end

Get all forks for a project.

 ==Parameters:
 [owner]  The user to which the project belongs
 [repo]  The repository/project to find forks for


def ensure_fork(owner, repo, fork_id)
      fork = retrieve_fork(owner, repo, fork_id)

      if fork.nil?
        warn "Could not retrieve fork #{owner}/#{repo} -> #{fork_id}"
        return
      end

      fork_name = if fork['full_name'].nil? then fork['url'].split(/\//)[4..5].join('/') else fork['full_name'] end
      fork_owner = fork_name.split(/\//)[0]
      fork_name = fork_name.split(/\//)[1]

      r = ensure_repo(fork_owner, fork_name, true)

      if r.nil?
        warn "Could not add #{fork_owner}/#{fork_name} as fork of #{owner}/#{repo}"
      else
        info "Added fork #{fork_owner}/#{fork_name} of #{owner}/#{repo}"
      end
      r
    end

Make sure that a fork is retrieved for a project


def ensure_issues(owner, repo)
      currepo = ensure_repo(owner, repo)
      if currepo.nil?
        warn "Could not find repo #{owner}/#{repo} for retrieving issues"
        return
      end

      issues = db[:issues].filter(:repo_id => currepo[:id]).all
      raw_issues = retrieve_issues(owner, repo).reduce([]) do |acc, x|
                     if issues.find { |y| y[:issue_id] == x['number'] }.nil?
                       acc << x
                     else
                         acc
                     end
                   end

      raw_issues.map { |x| save { ensure_issue(owner, repo, x['number']) } }.select { |x| !x.nil? }
    end

Make sure all issues exist for a project


def ensure_issue(owner, repo, issue_id, events = true,
                     comments = true, labels = true)

      issues = db[:issues]
      repository = ensure_repo(owner, repo)

      if repository.nil?
        warn "Could not find repo #{owner}/#{repo} for retrieving issue #{issue_id}"
        return
      end

      cur_issue = issues.first(:issue_id => issue_id,
                               :repo_id => repository[:id])

      retrieved = retrieve_issue(owner, repo, issue_id)

      if retrieved.nil?
        warn "Could not retrieve issue #{owner}/#{repo} -> #{issue_id}"
        return
      end

      # Pull requests and issues share the same issue_id
      pull_req = unless retrieved['pull_request'].nil? or
          retrieved['pull_request']['patch_url'].nil?
                   debug "Issue #{owner}/#{repo}->#{issue_id} is a pull request"
                   ensure_pull_request(owner, repo, issue_id, false, false, false)
                 end

      if cur_issue.nil?

        reporter = ensure_user(retrieved['user']['login'], false, false)
        assignee = unless retrieved['assignee'].nil?
                     ensure_user(retrieved['assignee']['login'], false, false)
                   end

        issues.insert(:repo_id => repository[:id],
                     :assignee_id => unless assignee.nil? then assignee[:id] end,
                     :reporter_id => reporter[:id],
                     :issue_id => issue_id,
                     :pull_request => if pull_req.nil? then false else true end,
                     :pull_request_id => unless pull_req.nil? then pull_req[:id] end,
                     :created_at => date(retrieved['created_at']))

        info "Added issue #{owner}/#{repo} -> #{issue_id}"
      else
        debug "Issue #{owner}/#{repo}->#{issue_id} exists"
        if cur_issue[:pull_request] == false and not pull_req.nil?
          info "Updated issue #{owner}/#{repo}->#{issue_id} as pull request"
          issues.filter(:issue_id => issue_id, :repo_id => repository[:id]).update(
              :pull_request => true,
              :pull_request_id => pull_req[:id])
        end
      end
      ensure_issue_events(owner, repo, issue_id) if events
      ensure_issue_comments(owner, repo, issue_id) if comments
      ensure_issue_labels(owner, repo, issue_id) if labels
      issues.first(:issue_id => issue_id,
                   :repo_id => repository[:id])
    end


def ensure_issue_events(owner, repo, issue_id)
      currepo = ensure_repo(owner, repo)

      if currepo.nil?
        warn "Could not find repository #{owner}/#{repo} for retrieving events for issue #{issue_id}"
        return
      end

      issue = ensure_issue(owner, repo, issue_id, false, false, false)
      if issue.nil?
        warn "Could not find issue #{owner}/#{repo} -> #{issue_id} for retrieving events"
        return
      end

      retrieve_issue_events(owner, repo, issue_id).reduce([]) do |acc, x|

        if db[:issue_events].first(:issue_id => issue[:id],
                                    :event_id => x['id']).nil?
          acc << x
        else
          acc
        end
      end.map { |x|
        save{ensure_issue_event(owner, repo, issue_id, x['id'])}
      }.select{|x| !x.nil?}
    end

Retrieve and process all events for an issue


def ensure_issue_event(owner, repo, issue_id, event_id)
      issue = ensure_issue(owner, repo, issue_id, false, false, false)

      if issue.nil?
        warn "Could not find issue #{owner}/#{repo} -> #{issue_id} for retrieving event #{event_id}"
        return
      end

      issue_event_str = "#{owner}/#{repo} -> #{issue_id}/#{event_id}"

      curevent = db[:issue_events].first(:issue_id => issue[:id],
                                          :event_id => event_id)
      if curevent.nil?

        retrieved = retrieve_issue_event(owner, repo, issue_id, event_id)

        if retrieved.nil?
          warn "Could not retrieve issue_event #{owner}/#{repo} -> #{issue_id}/#{issue_event_str}"
          return
        elsif retrieved['actor'].nil?
          warn "Could not find issue_event_actor #{owner}/#{repo} -> #{issue_id}/#{issue_event_str}"
          return
        end

        actor = ensure_user(retrieved['actor']['login'], false, false)

        action_specific = case retrieved['event']
                            when "referenced" then retrieved['commit_id']
                            when "merged" then retrieved['commit_id']
                            when "closed" then retrieved['commit_id']
                            else nil
                          end

        if retrieved['event'] == 'assigned'

          def update_assignee(owner, repo, issue, actor)
            db[:issues].first(:id => issue[:id]).update(:assignee_id => actor[:id])
            info "Updated #{owner}/#{repo} -> #{issue[:id]}, assignee -> #{actor[:id]}"
          end

          if issue[:assignee_id].nil? then
            update_assignee(owner, repo, issue, actor)
          else
            existing = db[:issue_events].\
                        filter(:issue_id => issue[:id],:action => 'assigned').\
                        order(Sequel.desc(:created_at)).first
            if existing.nil?
              update_assignee(owner, repo, issue, actor)
            elsif date(existing[:created_at]) < date(retrieved['created_at'])
              update_assignee(owner, repo, issue, actor)
            end
          end
        end

        db[:issue_events].insert(
            :event_id => event_id,
            :issue_id => issue[:id],
            :actor_id => unless actor.nil? then actor[:id] end,
            :action => retrieved['event'],
            :action_specific => action_specific,
            :created_at => date(retrieved['created_at']))

        info "Added issue_event #{owner}/#{repo} -> #{issue_id}/#{issue_event_str}"
        db[:issue_events].first(:issue_id => issue[:id],
                                 :event_id => event_id)
      else
        debug "Issue event #{owner}/#{repo} -> #{issue_id}/#{issue_event_str} exists"
        curevent
      end
    end

Retrieve and process +event_id+ for an +issue_id+

Retrieve and process + event_id + for an + issue_id +


def ensure_issue_comments(owner, repo, issue_id, pull_req_id = nil)
      currepo = ensure_repo(owner, repo)

      if currepo.nil?
        warn "Could not find repository #{owner}/#{repo} for retrieving issue comments for issue #{issue_id}"
        return
      end

      issue = if pull_req_id.nil?
                ensure_issue(owner, repo, issue_id, false, false, false)
              else
                db[:issues].first(:pull_request_id => pull_req_id)
              end

      if issue.nil?
        warn "Could not find issue #{owner}/#{repo} -> #{issue_id} for retrieving issue comments"
        return
      end

      retrieve_issue_comments(owner, repo, issue_id).reduce([]) do |acc, x|

        if db[:issue_comments].first(:issue_id => issue[:id],
                                    :comment_id => x['id']).nil?
          acc << x
        else
          acc
        end
      end.map { |x|
        save{ensure_issue_comment(owner, repo, issue_id, x['id'], pull_req_id)}
      }.select{|x| !x.nil?}
    end

Retrieve and process all comments for an issue.
 If pull_req_id is not nil this means that we are only retrieving
 comments for the pull request discussion for projects that don't have
 issues enabled

Retrieve and process all comments for an issue . If pull_req_id is not nil this means that we are only retrieving comments for the pull request discussion for projects that don t have issues enabled


def ensure_issue_comment(owner, repo, issue_id, comment_id, pull_req_id = nil)
      issue = if pull_req_id.nil?
                ensure_issue(owner, repo, issue_id, false, false, false)
              else
                db[:issues].first(:pull_request_id => pull_req_id)
              end

      if issue.nil?
        warn "Could not find issue #{owner}/#{repo} -> #{issue_id} for retrieving comment #{comment_id}"
        return
      end

      issue_comment_str = "#{owner}/#{repo} -> #{issue_id}/#{comment_id}"

      curcomment = db[:issue_comments].first(:issue_id => issue[:id],
                                          :comment_id => comment_id)
      if curcomment.nil?

        retrieved = retrieve_issue_comment(owner, repo, issue_id, comment_id)

        if retrieved.nil?
          warn "Could not retrieve issue_comment #{issue_comment_str}"
          return
        end

        user = ensure_user(retrieved['user']['login'], false, false)

        db[:issue_comments].insert(
            :comment_id => comment_id,
            :issue_id => issue[:id],
            :user_id => unless user.nil? then user[:id] end,
            :created_at => date(retrieved['created_at'])
        )

        info "Added issue_comment #{issue_comment_str}"
        db[:issue_comments].first(:issue_id => issue[:id],
                                   :comment_id => comment_id)
      else
        debug "Issue comment #{issue_comment_str} exists"
        curcomment
      end
    end

Retrieve and process +comment_id+ for an +issue_id+

Retrieve and process + comment_id + for an + issue_id +


def ensure_labels(owner, repo)
      currepo = ensure_repo(owner, repo)

      if currepo.nil?
        warn "Could not find #{owner}/#{repo} for retrieving issue labels"
        return
      end

      repo_labels = db[:repo_labels].filter(:repo_id => currepo[:id]).all

      retrieve_repo_labels(owner, repo).reduce([]) do |acc, x|
        if repo_labels.find {|y| y[:name] == x['name']}.nil?
          acc << x
        else
          acc
        end
      end.map { |x| save { ensure_repo_label(owner, repo, x['name']) } }.select { |x| !x.nil? }
    end


def ensure_repo_label(owner, repo, name)
      currepo = ensure_repo(owner, repo)

      if currepo.nil?
        warn "Could not find #{owner}/#{repo} for retrieving label #{name}"
        return
      end

      label = db[:repo_labels].first(:repo_id => currepo[:id], :name => name)

      if label.nil?
        retrieved = retrieve_repo_label(owner, repo, name)

        if retrieved.nil?
          warn "Could not retrieve repo_label #{owner}/#{repo} -> #{name}"
          return
        end

        db[:repo_labels].insert(
            :repo_id => currepo[:id],
            :name => name
        )

        info "Added repo_label #{owner}/#{repo} -> #{name}"
        db[:repo_labels].first(:repo_id => currepo[:id], :name => name)
      else
        label
      end
    end

Retrieve a single repository issue label by name


def ensure_issue_labels(owner, repo, issue_id)

      issue = ensure_issue(owner, repo, issue_id, false, false, false)

      if issue.nil?
        warn "Could not find issue #{owner}/#{repo} -> #{issue_id} for retrieving labels"
        return
      end

      issue_labels = db.from(:issue_labels, :repo_labels)\
                        .where(Sequel.qualify('issue_labels', 'label_id') => Sequel.qualify('repo_labels', 'id'))\
                        .where(Sequel.qualify('issue_labels', 'issue_id') => issue[:id])\
                        .select(Sequel.qualify('repo_labels', 'name')).all

      retrieve_issue_labels(owner, repo, issue_id).reduce([]) do |acc, x|
        if issue_labels.find {|y| y[:name] == x['name']}.nil?
          acc << x
        else
          acc
        end
      end.map { |x| save{ensure_issue_label(owner, repo, issue[:issue_id], x['name']) }}.select{|x| !x.nil?}

    end

Ensure that all labels have been assigned to the issue


def ensure_issue_label(owner, repo, issue_id, name)

      issue = ensure_issue(owner, repo, issue_id, false, false, false)

      if issue.nil?
        warn "Could not find issue #{owner}/#{repo} -> #{issue_id} to assign label #{name}"
        return
      end

      label = ensure_repo_label(owner, repo, name)

      if label.nil?
        warn "Could not find repo label #{owner}/#{repo} -> #{name}"
        return
      end

      issue_lbl = db[:issue_labels].first(:label_id => label[:id],
                                           :issue_id => issue[:id])

      if issue_lbl.nil?

        db[:issue_labels].insert(
            :label_id => label[:id],
            :issue_id => issue[:id],
        )
        info "Added issue_label #{name} to issue #{owner}/#{repo} -> #{issue_id}"
        db[:issue_labels].first(:label_id => label[:id],
                                 :issue_id => issue[:id])
      else
        debug "Issue label #{name} to issue #{owner}/#{repo} -> #{issue_id} exists"
        issue_lbl
      end

    end

Ensure that a specific label has been assigned to the issue


def transaction(&block)
      db
      persister

      result = nil
      start_time = Time.now
      begin
        db.transaction(:rollback => :reraise, :isolation => :repeatable,
                        :retry_on => @retry_on_error, :num_retries => 3) do
          result = yield block
        end
        total = Time.now.to_ms - start_time.to_ms
        debug "Transaction committed (#{total} ms)"
        result
      rescue StandardError => e
        total = Time.now.to_ms - start_time.to_ms
        warn "Transaction failed (#{total} ms)"
        raise e
      ensure
        GC.start
      end
    end

def
transaction
(
&
block
)
db
persister
result
=
start_time
=
Time
.
now
begin
db
.
transaction
(
:rollback
=>
:reraise
,
:isolation
=>
:repeatable
,
:retry_on
=>
@retry_on_error
,
:num_retries
=>
3
)
do
result
=
yield
block
end
total
=
Time
.
now
.
to_ms
start_time
.
to_ms
debug
"Transaction committed (#{total} ms)"
result
rescue
StandardError
=>
e
total
=
Time
.
now
.
to_ms
start_time
.
to_ms
warn
"Transaction failed (#{total} ms)"
raise
e
ensure
GC
.
start
end
end

Run a block in a DB transaction. Exceptions trigger transaction rollback
 and are rethrown.

Run a block in a DB transaction . Exceptions trigger transaction rollback and are rethrown .


def store_commit(c, repo, user)
      commits = db[:commits]
      commit = commits.first(:sha => c['sha'])

      if commit.nil?
        author = commit_user(c['author'], c['commit']['author'])
        commiter = commit_user(c['committer'], c['commit']['committer'])

        repository = ensure_repo(user, repo)

        if repository.nil?
          warn "Could not find repo #{user}/#{repo} for storing commit #{c['sha']}"
        end

        commits.insert(:sha => c['sha'],
                       :author_id => author[:id],
                       :committer_id => commiter[:id],
                       :project_id => if repository.nil? then nil else repository[:id] end ,
                       :created_at => date(c['commit']['author']['date'])
        )
        info "Added commit #{user}/#{repo} -> #{c['sha']} "
        commits.first(:sha => c['sha'])
      else
        debug "Commit #{user}/#{repo} -> #{c['sha']} exists"
        commit
      end
    end

Store a commit contained in a hash. First check whether the commit exists.

Store a commit contained in a hash . First check whether the commit exists .


def log(level, msg)

      case level
        when :fatal then
          loggerr.fatal (retrieve_caller + msg)
        when :error then
          loggerr.error (retrieve_caller + msg)
        when :warn then
          loggerr.warn  (retrieve_caller + msg)
        when :info then
          loggerr.info  (retrieve_caller + msg)
        when :debug then
          loggerr.debug (retrieve_caller + msg)
        else
          loggerr.debug (retrieve_caller + msg)
      end
    end

def log ( level , msg ) case level when :fatal then loggerr . fatal ( retrieve_caller + msg ) when :error then loggerr . error ( retrieve_caller + msg ) when :warn then loggerr . warn ( retrieve_caller + msg ) when :info then loggerr . info ( retrieve_caller + msg ) when :debug then loggerr . debug ( retrieve_caller + msg ) else loggerr . debug ( retrieve_caller + msg ) end end


def paged_api_request(url, pages = config(:mirror_history_pages_back),
                          last = nil)

      url = ensure_max_per_page(url)
      data = api_request_raw(url)

      return [] if data.nil?

      unless data.meta['link'].nil?
        links = parse_links(data.meta['link'])
        last = links['last'] if last.nil?

        if pages > 0
          pages = pages - 1
          if pages == 0
            return parse_request_result(data)
          end
        end

        if links['next'].nil?
          parse_request_result(data)
        else
          parse_request_result(data) | paged_api_request(links['next'], pages, last)
        end
      else
        parse_request_result(data)
      end
    end

def paged_api_request ( url , pages = config ( :mirror_history_pages_back ) , last = nil ) url = ensure_max_per_page ( url ) data = api_request_raw ( url ) return [ ] if data . nil? unless data . meta [ 'link' ] . nil? links = parse_links ( data . meta [ 'link' ] ) last = links [ 'last' ] if last . nil? if pages > 0 pages = pages - 1 if pages == 0 return parse_request_result ( data ) end end if links [ 'next' ] . nil? parse_request_result ( data ) else parse_request_result ( data ) | paged_api_request ( links [ 'next' ] , pages , last ) end else parse_request_result ( data ) end end

A paged request. Used when the result can expand to more than one
 result pages.

A paged request . Used when the result can expand to more than one result pages .


def last_updated(url, etag)
      begin
        ts = Time.now
        response = do_request(url, '', etag)
        info "Successful etag request. URL: #{url}, Etag: #{etag}, Remaining: #{@remaining}, Total: #{Time.now.to_ms - ts.to_ms} ms"
      rescue OpenURI::HTTPError => e
        response = e.io
        if response.status.first != '304'
          etag_request_error_message(url, e, etag)
          raise e
        end
      end

      return Time.parse(response.meta['last-modified']) unless response.meta['last-modified'].nil?
      return Time.at(86400)
    end

Check whether the resource identified by the provided url has changed


def num_pages(url)
      url = ensure_max_per_page(url)
      data = api_request_raw(url)

      if data.nil? or data.meta.nil? or data.meta['link'].nil?
        return 1
      end

      links = parse_links(data.meta['link'])

      if links.nil? or links['last'].nil?
        return 1
      end

      params = CGI::parse(URI::parse(links['last']).query)
      params['page'][0].to_i
    end

def num_pages ( url ) url = ensure_max_per_page ( url ) data = api_request_raw ( url ) if data . nil? or data . meta . nil? or data . meta [ 'link' ] . nil? return 1 end links = parse_links ( data . meta [ 'link' ] ) if links . nil? or links [ 'last' ] . nil? return 1 end params = CGI :: parse ( URI :: parse ( links [ 'last' ] ) . query ) params [ 'page' ] [ 0 ] . to_i end

Determine the number of pages contained in a multi-page API response

Determine the number of pages contained in a multi - page API response


def parse_links(links)
      links.split(/,/).reduce({}) do |acc, x|
        matches = x.strip.match(/<(.*)>; rel=\"(.*)\"/)
        acc[matches[2]] = matches[1]
        acc
      end
    end

def parse_links ( links ) links . split ( / / ) . reduce ( { } ) do | acc , x | matches = x . strip . match ( / \" \" / ) acc [ matches [ 2 ] ] = matches [ 1 ] acc end end


def parse_request_result(result)
      if result.nil?
        []
      else
        json = result.read

        if json.nil?
          []
        else
          r = JSON.parse(json)

          # Add the etag to the response only for individual entities
          if result.meta['etag'] and r.class != Array
            r['etag'] = result.meta['etag']
          end

          r
        end
      end
    end


def api_request_raw(url, media_type = '')

      begin
        start_time = Time.now

        contents = do_request(url, media_type)
        total = Time.now.to_ms - start_time.to_ms
        info "Successful request. URL: #{url}, Remaining: #{@remaining}, Total: #{total} ms"

        contents
      rescue OpenURI::HTTPError => e
        @remaining = e.io.meta['x-ratelimit-remaining'].to_i
        @reset = e.io.meta['x-ratelimit-reset'].to_i

        case e.io.status[0].to_i
          # The following indicate valid Github return codes
          when 400, # Bad request
              403, # Forbidden
              404, # Not found
              409, # Conflict -- returned on gets of empty repos
              422 then # Unprocessable entity
            warn request_error_msg(url, e)
            return nil
          when 401 # Unauthorized
            warn request_error_msg(url, e)
            warn "Unauthorised request with token: #{@token}"
            raise e
          when 451 # DMCA takedown
            warn request_error_msg(url, e)
            warn "Repo was taken down (DMCA)"
            return nil
          else # Server error or HTTP conditions that Github does not report
            warn request_error_msg(url, e)
            raise e
        end
      rescue StandardError => e
        warn error_msg(url, e)
        raise e
      ensure
        # The exact limit is only enforced upon the first @reset
        # No idea how many requests are available on this key. Sleep if we have run out
        if @remaining < @req_limit
          to_sleep = @reset - Time.now.to_i + 2
          warn "Request limit reached, reset in: #{to_sleep} secs"
          t = Thread.new do
            slept = 0
            while true do
              debug "Sleeping for #{to_sleep - slept} seconds"
              sleep 1
              slept += 1
            end
          end
          sleep([0, to_sleep].max)
          t.exit
        end
      end
    end

Do the actual request and return the result object


def attach_to(ip)
      TCPSocket.instance_eval do
        (class << self; self; end).instance_eval do
          alias_method :original_open, :open

          case RUBY_VERSION
            when /1.8/, /1.9/
              define_method(:open) do |conn_address, conn_port|
                original_open(conn_address, conn_port, ip)
              end
            else
              define_method(:open) do |conn_address, conn_port, local_host, local_port|
                original_open(conn_address, conn_port, ip, local_port)
              end
          end
        end
      end

      result = begin
        yield
      rescue StandardError => e
        raise e
      ensure
        TCPSocket.instance_eval do
          (class << self; self; end).instance_eval do
            alias_method :open, :original_open
            remove_method :original_open
          end
        end
      end

      result
    end

def attach_to ( ip ) TCPSocket . instance_eval do ( class << self ; self ; end ) . instance_eval do alias_method :original_open , :open case RUBY_VERSION when / / , / / define_method ( :open ) do | conn_address , conn_port | original_open ( conn_address , conn_port , ip ) end else define_method ( :open ) do | conn_address , conn_port , local_host , local_port | original_open ( conn_address , conn_port , ip , local_port ) end end end end result = begin yield rescue StandardError => e raise e ensure TCPSocket . instance_eval do ( class << self ; self ; end ) . instance_eval do alias_method :open , :original_open remove_method :original_open end end end result end

Attach to a specific IP address if the machine has multiple


def connect(adapter, settings)
      driver = ADAPTERS[adapter.intern]
      driver.new(settings)
    end

def connect ( adapter , settings ) driver = ADAPTERS [ adapter . intern ] driver . new ( settings ) end

Factory method for retrieving persistence connections.
 The +settings+ argument is a fully parsed YAML document
 passed on to adapters. The available +adapter+ are 'mongo' and 'noop'

Factory method for retrieving persistence connections . The + settings + argument is a fully parsed YAML document passed on to adapters . The available + adapter + are mongo and noop


def retrieve_commit(repo, sha, user)
      commit = persister.find(:commits, {'sha' => "#{sha}"})

      if commit.empty?
        url = ghurl "repos/#{user}/#{repo}/commits/#{sha}"
        c = api_request(url)

        if c.nil? or c.empty?
          return
        end

        # commit patches are big and not always interesting
        if config(:commit_handling) == 'trim'
          c['files'].each { |file| file.delete('patch') }
        end
        persister.store(:commits, c)
        info "Added commit #{user}/#{repo} -> #{sha}"
        c
      else
        debug "Commit #{user}/#{repo} -> #{sha} exists"
        commit.first
      end
    end


def retrieve_commits(repo, sha, user, pages = -1)

      url = if sha.nil?
              ghurl "repos/#{user}/#{repo}/commits"
            else
              ghurl "repos/#{user}/#{repo}/commits?sha=#{sha}"
            end

      commits = restricted_page_request(url, pages)

      commits.map do |c|
        retrieve_commit(repo, c['sha'], user)
      end.select{|x| not x.nil?}
    end

Retrieve commits starting from the provided +sha+

Retrieve commits starting from the provided + sha +


def retrieve_orgs(user)
      url = ghurl "users/#{user}/orgs"
      orgs = paged_api_request(url)
      orgs.map{|o| retrieve_org(o['login'])}
    end

def retrieve_orgs ( user ) url = ghurl "users/#{user}/orgs" orgs = paged_api_request ( url ) orgs . map { | o | retrieve_org ( o [ 'login' ] ) } end

Retrieve organizations the provided user participates into


def retrieve_watchers(user, repo)
      repo_bound_items(user, repo, :watchers,
                       ["repos/#{user}/#{repo}/stargazers"],
                       {'repo' => repo, 'owner' => user},
                       'login', item = nil, refresh = false, order = :desc)
    end

def retrieve_watchers ( user , repo ) repo_bound_items ( user , repo , :watchers , [ "repos/#{user}/#{repo}/stargazers" ] , { 'repo' => repo , 'owner' => user } , 'login' , item = nil , refresh = false , order = :desc ) end


def retrieve_watcher(user, repo, watcher)
      repo_bound_item(user, repo, watcher, :watchers,
                      ["repos/#{user}/#{repo}/stargazers"],
                      {'repo' => repo, 'owner' => user},
                      'login', order = :desc)
    end

def retrieve_watcher ( user , repo , watcher ) repo_bound_item ( user , repo , watcher , :watchers , [ "repos/#{user}/#{repo}/stargazers" ] , { 'repo' => repo , 'owner' => user } , 'login' , order = :desc ) end

Retrieve a single watcher for a repository


def get_repo_events(owner, repo)
      url = ghurl("repos/#{owner}/#{repo}/events")
      r = paged_api_request(url)

      r.each do |e|
        unless get_event(e['id']).empty?
          debug "Repository event #{owner}/#{repo} -> #{e['type']}-#{e['id']} already exists"
        else
          persister.store(:events, e)
          info "Added event for repository #{owner}/#{repo} -> #{e['type']}-#{e['id']}"
        end
      end

      persister.find(:events, {'repo.name' => "#{owner}/#{repo}"})

    end

Get all events for the specified repo.
 GitHub will only return 90 days of events

Get all events for the specified repo . GitHub will only return 90 days of events


def retrieve_master_branch_diff(owner, repo, branch, parent_owner, parent_repo, parent_branch)
      branch   = retrieve_default_branch(owner, repo) if branch.nil?
      parent_branch = retrieve_default_branch(parent_owner, parent_repo) if parent_branch.nil?
      return nil if branch.nil? or parent_branch.nil?

      cmp_url = "https://api.github.com/repos/#{parent_owner}/#{parent_repo}/compare/#{parent_branch}...#{owner}:#{branch}"
      api_request(cmp_url)
    end

Retrieve diff between two branches. If either branch name is not provided
 the branch name is resolved to the corresponding default branch

Retrieve diff between two branches . If either branch name is not provided the branch name is resolved to the corresponding default branch


def retrieve_default_branch(owner, repo, refresh = false)
      retrieved = retrieve_repo(owner, repo, refresh)
      return nil if retrieved.nil?

      master_branch = 'master'
      if retrieved['default_branch'].nil?
        # The currently stored repo entry has been created before the
        # default_branch field was added to the schema
        retrieved = retrieve_repo(owner, repo, true)
        return nil if retrieved.nil?
      end
      master_branch = retrieved['default_branch'] unless retrieved.nil?
      master_branch
    end

Retrieve the default branch for a repo. If nothing is retrieved, 'master' is returned

Retrieve the default branch for a repo . If nothing is retrieved master is returned


def process_options
      command = self
      @options = Trollop::options(command.args) do

        command.prepare_options(self)

        banner <<-END
Standard options:
        END

        opt :config, 'config.yaml file location', :short => 'c',
            :default => 'config.yaml'
        opt :verbose, 'verbose mode', :short => 'v'
        opt :addr, 'IP address to use for performing requests', :short => 'a',
            :type => String
        opt :token, 'GitHub OAuth token',
            :type => String, :short => 't'
        opt :req_limit, 'Number or requests to leave on any provided account (in reqs/hour)',
            :type => Integer, :short => 'l'
        opt :uniq, 'Unique name for this command. Will appear in logs.',
            :type => String, :short => 'u'
      end
    end

Specify and parse top-level command line options.

Specify and parse top - level command line options .


def validate
      if options[:config].nil?
        unless (File.exist?("config.yaml"))
          Trollop::die "No config file in default location (#{Dir.pwd}). You
                        need to specify the #{:config} parameter. Read the
                        documentation on how to create a config.yaml file."
        end
      else
        Trollop::die "Cannot find file #{options[:config]}" \
          unless File.exist?(options[:config])
      end

      unless @options[:user].nil?
        if not Process.uid == 0
          Trollop::die "Option --user (-u) can only be specified by root"
        end
          begin
            Etc.getpwnam(@options[:user])
          rescue ArgumentError
            Trollop::die "No such user: #{@options[:user]}"
          end
      end
    end

Examine the validity of the provided options in the context of the
 executed command. Subclasses can also call super to also invoke the checks
 provided by this class.

Examine the validity of the provided options in the context of the executed command . Subclasses can also call super to also invoke the checks provided by this class .


def queue_client(queue, key = queue, ack = :after, block)

      stopped = false
      while not stopped
        begin
          conn = Bunny.new(:host => config(:amqp_host),
                           :port => config(:amqp_port),
                           :username => config(:amqp_username),
                           :password => config(:amqp_password))
          conn.start

          ch  = conn.create_channel
          debug "Queue setting prefetch to #{config(:amqp_prefetch)}"
          ch.prefetch(config(:amqp_prefetch))
          debug "Queue connection to #{config(:amqp_host)} succeeded"

          x = ch.topic(config(:amqp_exchange), :durable => true,
                       :auto_delete => false)
          q = ch.queue(queue, :durable => true)
          q.bind(x, :routing_key => key)

          q.subscribe(:block => true,
                      :manual_ack => true) do |delivery_info, properties, msg|

            if ack == :before
              ch.acknowledge(delivery_info.delivery_tag)
            end

            begin
              block.call(msg)
            ensure
              if ack != :before
                ch.acknowledge(delivery_info.delivery_tag)
              end
            end
          end

        rescue Bunny::TCPConnectionFailed => e
          warn "Connection to #{config(:amqp_host)} failed. Retrying in 1 sec"
          sleep(1)
        rescue Bunny::PossibleAuthenticationFailureError => e
          warn "Could not authenticate as #{conn.username}"
        rescue Bunny::NotFound, Bunny::AccessRefused, Bunny::PreconditionFailed => e
          warn "Channel error: #{e}. Retrying in 1 sec"
          sleep(1)
        rescue Interrupt => _
          stopped = true
        rescue StandardError => e
          raise e
        end
      end

      ch.close unless ch.nil?
      conn.close unless conn.nil?

    end

Specify a handler to incoming messages from a connection to a queue.

 @param queue [String] the queue name to bind to
 @param key [String] routing key for msgs for binding the queue to the exchange.
 @param ack [Symbol] when should acks be send, :before or :after the block returns
 @param block [Block]: A block accepting one argument (the message)

Specify a handler to incoming messages from a connection to a queue .


def read_value(from, key)
      return from if key.nil? or key == ""

      key.split(/\./).reduce({}) do |acc, x|
        unless acc.nil?
          if acc.empty?
            # Initial run
            acc = from[x]
          else
            if acc.has_key?(x)
              acc = acc[x]
            else
              # Some intermediate key does not exist
              return nil
            end
          end
        else
          # Some intermediate key returned a null value
          # This indicates a malformed entry
          return nil
        end
      end
    end

Read the value for a key whose format is "foo.bar.baz" from a hierarchical
 map, where a dot represents one level deep in the hierarchy.

Read the value for a key whose format is foo . bar . baz from a hierarchical map where a dot represents one level deep in the hierarchy .


def location_filter(location)
      return nil if location.nil?
      location.\
        strip.\
        downcase.\
        tr('#"<>[]', '').\
        gsub(/^[0-9,\/().:]*/, '').\
        gsub(/ +/, ' ').\
        gsub(/,([a-z]*)/, '\1')
    end

def
location_filter
(
location
)
return
if
location
.
nil?
location
.
strip
.
downcase
.
tr
(
'#"<>[]'
,
''
)
.
gsub
(
/
\/
/
,
''
)
.
gsub
(
/
/
,
' '
)
.
gsub
(
/
/
,
'\1'
)
end

Standard filtering on all locations used by GHTorrent


def validate_usage!
      registered_topics = self.class.topics.map do |name, topic|
        topic.to_h.merge!(
          usage_count: messages_buffer[name]&.count || 0
        )
      end

      used_topics = messages_buffer.map do |name, usage|
        topic = self.class.topics[name] || Responders::Topic.new(name, registered: false)
        topic.to_h.merge!(usage_count: usage.count)
      end

      result = Karafka::Schemas::ResponderUsage.call(
        registered_topics: registered_topics,
        used_topics: used_topics
      )

      return if result.success?

      raise Karafka::Errors::InvalidResponderUsageError, result.errors
    end

def validate_usage! registered_topics = self . class . topics . map do | name , topic | topic . to_h . merge! ( usage_count : messages_buffer [ name ] &. count || 0 ) end used_topics = messages_buffer . map do | name , usage | topic = self . class . topics [ name ] || Responders :: Topic . new ( name , registered : false ) topic . to_h . merge! ( usage_count : usage . count ) end result = Karafka :: Schemas :: ResponderUsage . call ( registered_topics : registered_topics , used_topics : used_topics ) return if result . success? raise Karafka :: Errors :: InvalidResponderUsageError , result . errors end

Checks if we met all the topics requirements. It will fail if we didn't send a message to
 a registered required topic, etc.

Checks if we met all the topics requirements . It will fail if we didn t send a message to a registered required topic etc .


def validate_options!
      return true unless self.class.options_schema

      messages_buffer.each_value do |messages_set|
        messages_set.each do |message_data|
          result = self.class.options_schema.call(message_data.last)
          next if result.success?

          raise Karafka::Errors::InvalidResponderMessageOptionsError, result.errors
        end
      end
    end

def validate_options! return true unless self . class . options_schema messages_buffer . each_value do | messages_set | messages_set . each do | message_data | result = self . class . options_schema . call ( message_data . last ) next if result . success? raise Karafka :: Errors :: InvalidResponderMessageOptionsError , result . errors end end end

Checks if we met all the options requirements before sending them to the producer.

Checks if we met all the options requirements before sending them to the producer .


def deliver!
      messages_buffer.each_value do |data_elements|
        data_elements.each do |data, options|
          # We map this topic name, so it will match namespaced/etc topic in Kafka
          # @note By default will not change topic (if default mapper used)
          mapped_topic = Karafka::App.config.topic_mapper.outgoing(options[:topic])
          external_options = options.merge(topic: mapped_topic)
          producer(options).call(data, external_options)
        end
      end
    end

Takes all the messages from the buffer and delivers them one by one
 @note This method is executed after the validation, so we're sure that
   what we send is legit and it will go to a proper topics


def notice_signal(signal)
      Thread.new do
        Karafka.monitor.instrument('process.notice_signal', caller: self, signal: signal)
      end
    end

def notice_signal ( signal ) Thread . new do Karafka . monitor . instrument ( 'process.notice_signal' , caller : self , signal : signal ) end end

Informs monitoring about trapped signal
 @param [Symbol] signal type that we received
 @note We cannot perform logging from trap context, that's why
   we have to spin up a new thread to do this


def embedding_lookup(params, ids, partition_strategy: "mod", name: nil, validate_indices: true, max_norm: nil)
      _embedding_lookup_and_transform(params, ids, partition_strategy: partition_strategy, name: name, max_norm: max_norm, transform_fn: nil)
    end

def embedding_lookup ( params , ids , partition_strategy : "mod" , name : nil , validate_indices : true , max_norm : nil ) _embedding_lookup_and_transform ( params , ids , partition_strategy : partition_strategy , name : name , max_norm : max_norm , transform_fn : nil ) end

Looks up `ids` in a list of embedding tensors.

Looks up ids in a list of embedding tensors .


def _embedding_lookup_and_transform(params, ids, partition_strategy: "mod", name: nil, max_norm: nil, transform_fn: nil)
      raise TensorStream::ValueError, "Need at least one param" if params.nil?

      params = [params] unless params.is_a?(Array)

      TensorStream.name_scope(name, "embedding_lookup", values: params + [ids]) do |name|
        np = params.size
        ids = TensorStream.convert_to_tensor(ids, name: "ids")
        if (np == 1) && (transform_fn.nil? || (ids.shape.size == 1))
          result = nil
          TensorStream.colocate_with(params[0]) do
            result = _clip(TensorStream.gather(params[0], ids, name: name), ids, max_norm)
            result = transform_fn.call(result) if transform_fn
          end

          return TensorStream.identity(result)
        else
          flat_ids = TensorStream.reshape(ids, [-1])
          original_indices = TensorStream.range(TensorStream.size(flat_ids))

          p_assignments = nil
          new_ids = nil

          if partition_strategy == "mod"
            p_assignments = flat_ids % np
            new_ids = floor_div(flat_ids, np)
          elsif partition_strategy == "div"
            raise "not yet supported!"
          else
            raise TensorStream::ValueError, "Unrecognized partition strategy: " + partition_strategy
          end

          p_assignments = TensorStream.cast(p_assignments, :int32)
          gather_ids = TensorStream.dynamic_partition(new_ids, p_assignments, np)
          pindices = TensorStream.dynamic_partition(original_indices, p_assignments, np)
          partitioned_result = []
          (0...np).each do |p|
            pids = gather_ids[p]
            result = nil
            TensorStream.colocate_with(params[p]) do
              result = TensorStream.gather(params[p], pids)
              if transform_fn
                # If transform_fn is provided, the clip_by_norm precedes
                # the transform and hence must be co-located. See below
                # for the counterpart if transform_fn is not proveded.
                result = transform_fn.call(_clip(result, pids, max_norm))
              end
            end
            partitioned_result << result
          end
          ret = TensorStream.dynamic_stitch(pindices, partitioned_result, name: name)

          if transform_fn.nil?
            element_shape_s = params[0].shape[1..-1]
            params[1..-1].each { |p| element_shape_s = element_shape_s.merge_with(p.shape[1..-1]) }
          else
            element_shape_s = ret.shape[1..-1]
          end

           # Compute the dynamic element shape.
          element_shape_d = if element_shape_s.fully_defined?
                               element_shape_s
                            elsif transform_fn.nil?
                              # It's important that we compute params[0].shape on the right device
                              # to avoid data motion.
                              TensorStream.colocate_with(params[0]) do
                                params_shape = TensorStream.shape(params[0])
                                params_shape[1..-1]
                              end
                            else
                              TensorStream.shape(ret)[1..-1]
                            end
          ret = TensorStream.reshape(ret, TensorStream.concat([TensorStream.shape(ids), element_shape_d], 0))
          ret = _clip(ret, ids, max_norm) unless transform_fn
          ret
        end
      end
    end

Helper function for embedding_lookup and _compute_sampled_logits.

Helper function for embedding_lookup and _compute_sampled_logits .


def load(pbfile)
      f = File.new(pbfile, "r")
      lines = []
      while !f.eof? && (str = f.readline.strip)
        lines << str
      end
      evaluate_lines(lines)
    end

def load ( pbfile ) f = File . new ( pbfile , "r" ) lines = [ ] while ! f . eof? && ( str = f . readline . strip ) lines << str end evaluate_lines ( lines ) end

parsers a protobuf file and spits out
 a ruby hash


def assert_equal(x, y, data: nil, summarize: nil, message: nil, name: nil)
      _op(:assert_equal, x, y, data: data, summarize: summarize, message: message, name: name)
    end

def assert_equal ( x , y , data : nil , summarize : nil , message : nil , name : nil ) _op ( :assert_equal , x , y , data : data , summarize : summarize , message : message , name : name ) end

Assert the condition x == y holds element-wise.

 Argmuments

 +x+ Numeric Tensor.
 +y+ Numeric Tensor, same dtype as and broadcastable to x.

 Returns
 Op that raises InvalidArgumentError if x == y is false

Assert the condition x == y holds element - wise .


def gradients(tensor_ys, wrt_xs, name: "gradients", stop_gradients: nil)
      tensor_ys = tensor_ys.op
      gs = wrt_xs.map(&:op).collect { |x|
        stops = stop_gradients ? stop_gradients.map(&:name).join("_") : ""
        gradient_program_name = "grad_#{tensor_ys.name}_#{x.name}_#{stops}".to_sym
        tensor_graph = tensor_ys.graph

        tensor_program = if tensor_graph.node_added?(gradient_program_name)
          tensor_graph.get_node(gradient_program_name)
        else
          tensor_graph.name_scope("gradient_wrt_#{x.name}") do
            derivative_ops = TensorStream::MathGradients.derivative(tensor_ys, x, graph: tensor_graph,
                                                                                stop_gradients: stop_gradients)
            tensor_graph.add_node!(gradient_program_name, derivative_ops)
          end
        end
        tensor_program
      }

      gs
    end

Constructs symbolic derivatives of ys of input w.r.t. x in wrt_xs.

 ys and xs are each a Tensor or a list of tensors. grad_ys is a list of Tensor, holding the gradients received by the ys. The list must be the same length as ys.

 Arguments:
 +tensor_ys+ : A Tensor or list of tensors to be differentiated.
 +wrt_xs+ : A Tensor or list of tensors to be used for differentiation.
 +stop_gradients+ :  Optional. A Tensor or list of tensors not to differentiate through

Constructs symbolic derivatives of ys of input w . r . t . x in wrt_xs .


def random_normal(shape, dtype: :float32, mean: 0.0, stddev: 1.0, seed: nil, name: nil)
      options = {dtype: dtype, mean: mean, stddev: stddev, seed: seed, name: name}
      _op(:random_standard_normal, shape, options)
    end

def random_normal ( shape , dtype : :float32 , mean : 0.0 , stddev : 1.0 , seed : nil , name : nil ) options = { dtype : dtype , mean : mean , stddev : stddev , seed : seed , name : name } _op ( :random_standard_normal , shape , options ) end

Outputs random values from a normal distribution.

Outputs random values from a normal distribution .


def eye(num_rows, num_columns: nil, dtype: :float32, name: nil)
      _op(:eye, num_rows, num_columns || num_rows, data_type: dtype, name: name)
    end

def eye ( num_rows , num_columns : nil , dtype : :float32 , name : nil ) _op ( :eye , num_rows , num_columns || num_rows , data_type : dtype , name : name ) end


def glorot_uniform_initializer(seed: nil, dtype: nil)
      TensorStream::Initializer.new(-> { _op(:glorot_uniform, seed: seed, data_type: dtype) })
    end

def glorot_uniform_initializer ( seed : nil , dtype : nil ) TensorStream :: Initializer . new ( -> { _op ( :glorot_uniform , seed : seed , data_type : dtype ) } ) end

The Glorot uniform initializer, also called Xavier uniform initializer.

 It draws samples from a uniform distribution within [-limit, limit]
 where limit is sqrt(6 / (fan_in + fan_out)) where fan_in is the number
 of input units in the weight tensor and fan_out is the number of output units in the weight tensor.

The Glorot uniform initializer also called Xavier uniform initializer .


def random_uniform_initializer(minval: 0, maxval: 1, seed: nil, dtype: nil)
      TensorStream::Initializer.new(-> { _op(:random_uniform, minval: 0, maxval: 1, seed: seed, data_type: dtype) })
    end

def random_uniform_initializer ( minval : 0 , maxval : 1 , seed : nil , dtype : nil ) TensorStream :: Initializer . new ( -> { _op ( :random_uniform , minval : 0 , maxval : 1 , seed : seed , data_type : dtype ) } ) end

Initializer that generates tensors with a uniform distribution.

Initializer that generates tensors with a uniform distribution .


def slice(input, start, size, name: nil)
      _op(:slice, input, start, size: size, name: name)
    end

def slice ( input , start , size , name : nil ) _op ( :slice , input , start , size : size , name : name ) end

Extracts a slice from a tensor.

 This operation extracts a slice of size size from a tensor input starting at the location specified by begin.
 The slice size is represented as a tensor shape, where size[i] is the number of elements of the 'i'th dimension of input that you want to slice. The starting location (begin) for the slice is
 represented as an offset in each dimension of input. In other words, begin[i] is the offset into the 'i'th dimension of input that you want to slice from.


def ones(shape, dtype: :float32, name: nil)
      _op(:ones, shape, data_type: dtype, name: name)
    end

def ones ( shape , dtype : :float32 , name : nil ) _op ( :ones , shape , data_type : dtype , name : name ) end

Creates a tensor with all elements set to 1.

Creates a tensor with all elements set to 1 .


def logical_and(input_a, input_b, name: nil)
      check_data_types(input_a, input_b)
      _op(:logical_and, input_a, input_b, name: name)
    end

Returns the truth value of x AND y element-wise.

Returns the truth value of x AND y element - wise .


def reduce_mean(input_tensor, axis = nil, keepdims: false, name: nil)
      reduce(:mean, input_tensor, axis, keepdims: keepdims, name: name)
    end

def reduce_mean ( input_tensor , axis = nil , keepdims : false , name : nil ) reduce ( :mean , input_tensor , axis , keepdims : keepdims , name : name ) end

Computes the mean of elements across dimensions of a tensor.

Computes the mean of elements across dimensions of a tensor .


def concat(values, axis, name: "concat")
      if values.is_a?(Array)
        _op(:concat, axis, *values, name: name)
      else
        _op(:concat, axis, values, name: name)
      end
    end

def concat ( values , axis , name : "concat" ) if values . is_a? ( Array ) _op ( :concat , axis , values , name : name ) else _op ( :concat , axis , values , name : name ) end end

Concatenates tensors along one dimension.

Concatenates tensors along one dimension .


def dynamic_partition(data, partitions, num_partitions, name: nil)
      result = _op(:dynamic_partition, data, partitions, num_partitions: num_partitions, name: nil)
      num_partitions.times.map do |index|
        result[index]
      end
    end

def dynamic_partition ( data , partitions , num_partitions , name : nil ) result = _op ( :dynamic_partition , data , partitions , num_partitions : num_partitions , name : nil ) num_partitions . times . map do | index | result [ index ] end end

Partitions data into num_partitions tensors using indices from partitions


def where(condition, true_t = nil, false_t = nil, name: nil)
      _op(:where, condition, true_t, false_t, name: name)
    end

def where ( condition , true_t = nil , false_t = nil , name : nil ) _op ( :where , condition , true_t , false_t , name : name ) end

Return the elements, either from x or y, depending on the condition.

Return the elements either from x or y depending on the condition .


def asin(input, name: nil)
      check_allowed_types(input, FLOATING_POINT_TYPES)
      _op(:asin, input, name: name)
    end

def asin ( input , name : nil ) check_allowed_types ( input , FLOATING_POINT_TYPES ) _op ( :asin , input , name : name ) end


def acos(input, name: nil)
      check_allowed_types(input, FLOATING_POINT_TYPES)
      _op(:acos, input, name: name)
    end

def acos ( input , name : nil ) check_allowed_types ( input , FLOATING_POINT_TYPES ) _op ( :acos , input , name : name ) end


def atan(input, name: nil)
      check_allowed_types(input, FLOATING_POINT_TYPES)
      _op(:atan, input, name: name)
    end

def atan ( input , name : nil ) check_allowed_types ( input , FLOATING_POINT_TYPES ) _op ( :atan , input , name : name ) end


def cast(input, dtype, name: nil)
      input = convert_to_tensor(input)
      return input if input.data_type == dtype

      _op(:cast, input, data_type: dtype, name: name)
    end

def cast ( input , dtype , name : nil ) input = convert_to_tensor ( input ) return input if input . data_type == dtype _op ( :cast , input , data_type : dtype , name : name ) end


def print(input, data, message: nil, name: nil)
      _op(:print, input, data, message: message, name: name)
    end

def print ( input , data , message : nil , name : nil ) _op ( :print , input , data , message : message , name : name ) end

Prints a list of tensors.

 This is an identity op (behaves like tf.identity) with the side effect of printing data when evaluating.


def sec(input, name: nil)
      check_allowed_types(input, FLOATING_POINT_TYPES)
      _op(:sec, input, name: name)
    end

def sec ( input , name : nil ) check_allowed_types ( input , FLOATING_POINT_TYPES ) _op ( :sec , input , name : name ) end


def sqrt(input, name: nil)
      check_allowed_types(input, FLOATING_POINT_TYPES)
      _op(:sqrt, input, name: name)
    end

def sqrt ( input , name : nil ) check_allowed_types ( input , FLOATING_POINT_TYPES ) _op ( :sqrt , input , name : name ) end


def log(input, name: nil)
      check_allowed_types(input, FLOATING_POINT_TYPES)
      _op(:log, input, name: name)
    end

def log ( input , name : nil ) check_allowed_types ( input , FLOATING_POINT_TYPES ) _op ( :log , input , name : name ) end

Computes natural logarithm of x element-wise.

Computes natural logarithm of x element - wise .


def exp(input, name: nil)
      check_allowed_types(input, FLOATING_POINT_TYPES)
      _op(:exp, input, name: name)
    end

def exp ( input , name : nil ) check_allowed_types ( input , FLOATING_POINT_TYPES ) _op ( :exp , input , name : name ) end

Computes exponential of x element - wise .


def pad(tensor, paddings, mode: "CONSTANT", name: nil)
      _op(:pad, tensor, paddings, mode: mode, name: name)
    end

def pad ( tensor , paddings , mode : "CONSTANT" , name : nil ) _op ( :pad , tensor , paddings , mode : mode , name : name ) end

Pads a tensor.
 This operation pads a tensor according to the paddings you specify.

Pads a tensor . This operation pads a tensor according to the paddings you specify .


def gather(params, indices, validate_indices: nil,
      name: nil,
      axis: 0)
      _op(:gather, params, indices, validate_indices: validate_indices, name: name, axis: axis)
    end

def gather ( params , indices , validate_indices : nil , name : nil , axis : 0 ) _op ( :gather , params , indices , validate_indices : validate_indices , name : name , axis : axis ) end

Gather slices from params and axis according to indices.

Gather slices from params and axis according to indices .


def pack(values, axis: 0, name: "pack")
      _op(:stack, *values, axis: axis, name: name)
    end

def pack ( values , axis : 0 , name : "pack" ) _op ( :stack , values , axis : axis , name : name ) end


def unpack(value, num: nil, axis: 0, name: "unpack")
      unstack(value, num: num, axis: axis, name: name)
    end

def unpack ( value , num : nil , axis : 0 , name : "unpack" ) unstack ( value , num : num , axis : axis , name : name ) end


def case(args = {})
      args = args.dup
      default = args.delete(:default)
      exclusive = args.delete(:exclusive)
      strict = args.delete(:strict)
      name = args.delete(:name)

      predicates = []
      functions = []

      args.each do |k, v|
        raise "Invalid argment or option #{k}" unless k.is_a?(Tensor)

        predicates << k
        functions << (v.is_a?(Proc) ? v.call : v)
      end

      _op(:case, predicates, default, *functions, exclusive: exclusive, strict: strict, name: name)
    end

def
case
(
args
=
{
}
)
args
=
args
.
dup
default
=
args
.
delete
(
:default
)
exclusive
=
args
.
delete
(
:exclusive
)
strict
=
args
.
delete
(
:strict
)
name
=
args
.
delete
(
:name
)
predicates
=
[
]
functions
=
[
]
args
.
each
do
|
k
,
v
|
raise
"Invalid argment or option #{k}"
unless
k
.
is_a?
(
Tensor
)
predicates
<<
k
functions
<<
(
v
.
is_a?
(
Proc
)
v
.
call
:
v
)
end
_op
(
:case
,
predicates
,
default
,
functions
,
exclusive
:
exclusive
,
strict
:
strict
,
name
:
name
)
end

Create a case operation.

 The pred_fn_pairs parameter is a dict or list of pairs of size N.
 Each pair contains a boolean scalar tensor and a proc that creates the tensors to be returned if the boolean evaluates to true.
 default is a proc generating a list of tensors. All the proc in pred_fn_pairs as well as default (if provided) should return the
 same number and types of tensors.


def i_op(code, *args)
      options = if args.last.is_a?(Hash)
        args.pop
      else
        {}
      end

      args << options.merge(internal: true)
      Graph.get_default_graph.add_op!(code.to_sym, *args)
    end

def i_op ( code , * args ) options = if args . last . is_a? ( Hash ) args . pop else { } end args << options . merge ( internal : true ) Graph . get_default_graph . add_op! ( code . to_sym , args ) end

same as op but with a marker that it was internal generated


def broadcast_dimensions(input, dims = [])
      return input if dims.empty?

      d = dims.shift

      if input.is_a?(Array) && (get_rank(input) - 1) == dims.size
        row_to_dup = input.collect { |item|
          broadcast_dimensions(item, dims.dup)
        }

        row_to_dup + Array.new(d) { row_to_dup }.flatten(1)
      elsif input.is_a?(Array)
        Array.new(d) { broadcast_dimensions(input, dims.dup) }
      else
        Array.new(d + 1) { input }
      end
    end

def broadcast_dimensions ( input , dims = [ ] ) return input if dims . empty? d = dims . shift if input . is_a? ( Array ) && ( get_rank ( input ) - 1 ) == dims . size row_to_dup = input . collect { | item | broadcast_dimensions ( item , dims . dup ) } row_to_dup + Array . new ( d ) { row_to_dup } . flatten ( 1 ) elsif input . is_a? ( Array ) Array . new ( d ) { broadcast_dimensions ( input , dims . dup ) } else Array . new ( d + 1 ) { input } end end


def vector_op(vector, vector2, switch = false, safe = true, &block)
      if get_rank(vector) < get_rank(vector2) # upgrade rank of A
        duplicated = Array.new(vector2.size) {
          vector
        }
        return vector_op(duplicated, vector2, switch, &block)
      end

      return yield(vector, vector2) unless vector.is_a?(Array)

      vector.each_with_index.collect { |input, index|
        next vector_op(input, vector2, switch, &block) if input.is_a?(Array) && get_rank(vector) > get_rank(vector2)

        if safe && vector2.is_a?(Array)
          next nil if vector2.size != 1 && index >= vector2.size
        end

        z = if vector2.is_a?(Array)
          if index < vector2.size
            vector2[index]
          else
            raise "incompatible tensor shapes used during op" if vector2.size != 1
            vector2[0]
          end
        else
          vector2
        end

        if input.is_a?(Array)
          vector_op(input, z, switch, &block)
        else
          switch ? yield(z, input) : yield(input, z)
        end
      }.compact
    end


def transpose_with_perm(arr, new_arr, shape, new_shape, perm)
      arr_size = shape.reduce(:*)
      divisors = shape.dup.drop(1).reverse.inject([1]) { |a, s|
        a << s * a.last
      }.reverse

      multipliers = new_shape.dup.drop(1).reverse.inject([1]) { |a, s|
        a << s * a.last
      }.reverse

      arr_size.times do |p|
        ptr = p
        index = []
        divisors.each_with_object(index) do |div, a|
          a << (ptr / div.to_f).floor
          ptr = ptr % div
        end

        # remap based on perm
        remaped = perm.map { |x| index[x] }

        ptr2 = 0
        multipliers.each_with_index do |m, idx|
          ptr2 += remaped[idx] * m
        end

        new_arr[ptr2] = arr[p]
      end

      [new_arr, new_shape]
    end

def
transpose_with_perm
(
arr
,
new_arr
,
shape
,
new_shape
,
perm
)
arr_size
=
shape
.
reduce
(
:*
)
divisors
=
shape
.
dup
.
drop
(
1
)
.
reverse
.
inject
(
[
1
]
)
{
|
a
,
s
|
a
<<
s
*
a
.
last
}
.
reverse
multipliers
=
new_shape
.
dup
.
drop
(
1
)
.
reverse
.
inject
(
[
1
]
)
{
|
a
,
s
|
a
<<
s
*
a
.
last
}
.
reverse
arr_size
.
times
do
|
p
|
ptr
=
p
index
=
[
]
divisors
.
each_with_object
(
index
)
do
|
div
,
a
|
a
<<
(
ptr
/
div
.
to_f
)
.
floor
ptr
=
ptr
%
div
end
# remap based on perm
remaped
=
perm
.
map
{
|
x
|
index
[
x
]
}
ptr2
=
0
multipliers
.
each_with_index
do
|
m
,
idx
|
ptr2
+=
remaped
[
idx
]
*
m
end
new_arr
[
ptr2
]
=
arr
[
p
]
end
[
new_arr
,
new_shape
]
end

general case transposition with flat arrays


def add(input_a, input_b, name: nil)
      input_a, input_b = apply_data_type_coercion(input_a, input_b)
      _op(:add, input_a, input_b, name: name)
    end

def add ( input_a , input_b , name : nil ) input_a , input_b = apply_data_type_coercion ( input_a , input_b ) _op ( :add , input_a , input_b , name : name ) end

Returns x + y element-wise.

 This operation supports broadcasting

 @param input_a tensor X
 @param input_b tensor Y

 Options:
 @option name Optional name
 @return Tensor


def argmax(input_a, axis = nil, name: nil, dimension: nil, output_type: :int32)
      check_allowed_types(input_a, TensorStream::Ops::NUMERIC_TYPES)
      check_allowed_types(axis, TensorStream::Ops::INTEGER_TYPES)
      _op(:argmax, input_a, axis, name: name, dimension: dimension, output_type: output_type)
    end

def argmax ( input_a , axis = nil , name : nil , dimension : nil , output_type : :int32 ) check_allowed_types ( input_a , TensorStream :: Ops :: NUMERIC_TYPES ) check_allowed_types ( axis , TensorStream :: Ops :: INTEGER_TYPES ) _op ( :argmax , input_a , axis , name : name , dimension : dimension , output_type : output_type ) end

Returns the index with the largest value across axes of a tensor.

 @param input_a tensor X (of type NUMERIC_TYPES)
 @param axis Describes which axis of the input tensor to reduce across. For vectors, use axis = 0 (of type INTEGER_TYPES)

 Options:
 @option name Optional name
 @option dimension Same as axis
 @option output_type Output data type defaults to int32 default (:int32)
 @return Tensor

Returns the index with the largest value across axes of a tensor .


def ceil(input_a, name: nil)
      check_allowed_types(input_a, TensorStream::Ops::FLOATING_POINT_TYPES)
      _op(:ceil, input_a, name: name)
    end

def ceil ( input_a , name : nil ) check_allowed_types ( input_a , TensorStream :: Ops :: FLOATING_POINT_TYPES ) _op ( :ceil , input_a , name : name ) end

Returns element-wise smallest integer in not less than x

 @param input_a tensor X (of type FLOATING_POINT_TYPES)

 Options:
 @option name Optional name
 @return Tensor

Returns element - wise smallest integer in not less than x


def cos(input_a, name: nil)
      check_allowed_types(input_a, TensorStream::Ops::FLOATING_POINT_TYPES)
      _op(:cos, input_a, name: name)
    end

def cos ( input_a , name : nil ) check_allowed_types ( input_a , TensorStream :: Ops :: FLOATING_POINT_TYPES ) _op ( :cos , input_a , name : name ) end

Computes cos of input element-wise.

 @param input_a tensor X (of type FLOATING_POINT_TYPES)

 Options:
 @option name Optional name
 @return Tensor


def floor(input_a, name: nil)
      check_allowed_types(input_a, TensorStream::Ops::FLOATING_POINT_TYPES)
      _op(:floor, input_a, name: name)
    end

def floor ( input_a , name : nil ) check_allowed_types ( input_a , TensorStream :: Ops :: FLOATING_POINT_TYPES ) _op ( :floor , input_a , name : name ) end

Returns element-wise largest integer not greater than x.

 @param input_a tensor X (of type FLOATING_POINT_TYPES)

 Options:
 @option name Optional name
 @return Tensor

Returns element - wise largest integer not greater than x .


def mod(input_a, input_b, name: nil)
      input_a, input_b = apply_data_type_coercion(input_a, input_b)
      _op(:mod, input_a, input_b, name: name)
    end

def mod ( input_a , input_b , name : nil ) input_a , input_b = apply_data_type_coercion ( input_a , input_b ) _op ( :mod , input_a , input_b , name : name ) end

Returns element-wise remainder of division.

 This operation supports broadcasting

 @param input_a tensor X
 @param input_b tensor Y

 Options:
 @option name Optional name
 @return Tensor

Returns element - wise remainder of division .


def pow(input_a, input_b, name: nil)
      input_a, input_b = apply_data_type_coercion(input_a, input_b)
      _op(:pow, input_a, input_b, name: name)
    end

def pow ( input_a , input_b , name : nil ) input_a , input_b = apply_data_type_coercion ( input_a , input_b ) _op ( :pow , input_a , input_b , name : name ) end

Computes the power of one value to another X^Y element wise

 This operation supports broadcasting

 @param input_a tensor X
 @param input_b tensor Y

 Options:
 @option name Optional name
 @return Tensor


def prod(input_a, axis = nil, name: nil, keepdims: false)
      check_allowed_types(axis, TensorStream::Ops::INTEGER_TYPES)
      input_a = TensorStream.convert_to_tensor(input_a)
      return input_a if input_a.shape.scalar?
      axis = cast_axis(input_a, axis)
      _op(:prod, input_a, axis, name: name, keepdims: keepdims)
    end

def prod ( input_a , axis = nil , name : nil , keepdims : false ) check_allowed_types ( axis , TensorStream :: Ops :: INTEGER_TYPES ) input_a = TensorStream . convert_to_tensor ( input_a ) return input_a if input_a . shape . scalar? axis = cast_axis ( input_a , axis ) _op ( :prod , input_a , axis , name : name , keepdims : keepdims ) end

Computes the product of elements across dimensions of a tensor.
 Reduces input_tensor along the dimensions given in axis. Unless keepdims is true, the rank of the
 tensor is reduced by 1 for each entry in axis. If keepdims is true, the reduced dimensions are
 retained with length 1.
 If axis has no entries, all dimensions are reduced, and a tensor with a single element is returned.

 @param input_a tensor X
 @param axis tensor X (of type INTEGER_TYPES)

 Options:
 @option name Optional name
 @option keepdims If true, retains reduced dimensions with length 1. default (false)
 @return Tensor

Computes the product of elements across dimensions of a tensor . Reduces input_tensor along the dimensions given in axis . Unless keepdims is true the rank of the tensor is reduced by 1 for each entry in axis . If keepdims is true the reduced dimensions are retained with length 1 . If axis has no entries all dimensions are reduced and a tensor with a single element is returned .


def random_uniform(shape, name: nil, dtype: :float32, minval: 0, maxval: 1, seed: nil)
      _op(:random_uniform, shape, name: name, dtype: dtype, minval: minval, maxval: maxval, seed: seed)
    end

def random_uniform ( shape , name : nil , dtype : :float32 , minval : 0 , maxval : 1 , seed : nil ) _op ( :random_uniform , shape , name : name , dtype : dtype , minval : minval , maxval : maxval , seed : seed ) end

Outputs random values from a uniform distribution.

 @param shape A 1-D integer Tensor or array. The shape of the output tensor.

 Options:
 @option name Optional name
 @option dtype The type of the output: float16, float32, float64, int32, or int64 default (:float32)
 @option minval A 0-D Tensor or ruby value of type dtype. The lower bound on the range of random values to generate. Defaults to 0. default (0)
 @option maxval A 0-D Tensor or ruby value of type dtype. The upper bound on the range of random values to generate. Defaults to 1 if dtype is floating point. default (1)
 @option seed  A ruby integer. Used to create a random seed for the distribution. See set_random_seed for behavior.
 @return Tensor

Outputs random values from a uniform distribution .


def range(start = 0, limit = 0, delta = 1, name: "range", dtype: nil, output_type: :int32)
      _op(:range, start, limit, delta, name: name, dtype: dtype, output_type: output_type)
    end

def range ( start = 0 , limit = 0 , delta = 1 , name : "range" , dtype : nil , output_type : :int32 ) _op ( :range , start , limit , delta , name : name , dtype : dtype , output_type : output_type ) end

Creates a sequence of numbers.
 Creates a sequence of numbers that begins at start and extends by increments of delta up to but not including limit.

 @param start Acts as first entry in the range if limit is not nil; otherwise, acts as range limit and first entry defaults to 0.
 @param limit Upper limit of sequence, exclusive. If nil, defaults to the value of start while the first entry of the range defaults to 0.
 @param delta Number that increments start. Defaults to 1.

 Options:
 @option name  A name for the operation. Defaults to "range". default ("range")
 @option dtype The type of the elements of the resulting tensor.
 @option output_type Output data type defaults to int32 default (:int32)
 @return Tensor

Creates a sequence of numbers . Creates a sequence of numbers that begins at start and extends by increments of delta up to but not including limit .


def rank(input, name: nil)
      input = convert_to_tensor(input)
      return cons(input.shape.ndims) if input.shape.known?
      _op(:rank, input, name: name)
    end

def rank ( input , name : nil ) input = convert_to_tensor ( input ) return cons ( input . shape . ndims ) if input . shape . known? _op ( :rank , input , name : name ) end

Returns the rank of a tensor

 @param input A tensor

 Options:
 @option name Optional name
 @return Tensor


def round(input_a, name: nil)
      check_allowed_types(input_a, TensorStream::Ops::FLOATING_POINT_TYPES)
      _op(:round, input_a, name: name)
    end

def round ( input_a , name : nil ) check_allowed_types ( input_a , TensorStream :: Ops :: FLOATING_POINT_TYPES ) _op ( :round , input_a , name : name ) end

Rounds the values of a tensor to the nearest integer, element-wise

 @param input_a tensor X (of type FLOATING_POINT_TYPES)

 Options:
 @option name Optional name
 @return Tensor

Rounds the values of a tensor to the nearest integer element - wise


def rsqrt(input_a, name: nil)
      check_allowed_types(input_a, TensorStream::Ops::FLOATING_POINT_TYPES)
      _op(:rsqrt, input_a, name: name)
    end

def rsqrt ( input_a , name : nil ) check_allowed_types ( input_a , TensorStream :: Ops :: FLOATING_POINT_TYPES ) _op ( :rsqrt , input_a , name : name ) end

Computes reciprocal of square root of x element-wise.

 @param input_a tensor X (of type FLOATING_POINT_TYPES)

 Options:
 @option name Optional name
 @return Tensor

Computes reciprocal of square root of x element - wise .


def shape(input, name: nil, out_type: :int32)
      return constant(shape_eval(input, out_type), dtype: out_type, name: "Shape/#{name}") if input.is_a?(Array) && !input[0].is_a?(Tensor)
      return constant(input.shape.shape, dtype: out_type, name: "Shape/#{input.name}_c") if shape_full_specified(input)
      _op(:shape, input, name: name, out_type: out_type)
    end

def shape ( input , name : nil , out_type : :int32 ) return constant ( shape_eval ( input , out_type ) , dtype : out_type , name : "Shape/#{name}" ) if input . is_a? ( Array ) && ! input [ 0 ] . is_a? ( Tensor ) return constant ( input . shape . shape , dtype : out_type , name : "Shape/#{input.name}_c" ) if shape_full_specified ( input ) _op ( :shape , input , name : name , out_type : out_type ) end

This operation returns a 1-D integer tensor representing the shape of input

 @param input A tensor

 Options:
 @option name Optional name
 @option out_type Optional output type default (:int32)
 @return Tensor

This operation returns a 1 - D integer tensor representing the shape of input


def sigmoid(input_a, name: nil)
      check_allowed_types(input_a, TensorStream::Ops::FLOATING_POINT_TYPES)
      _op(:sigmoid, input_a, name: name)
    end

def sigmoid ( input_a , name : nil ) check_allowed_types ( input_a , TensorStream :: Ops :: FLOATING_POINT_TYPES ) _op ( :sigmoid , input_a , name : name ) end

Computes sigmoid of x element-wise.

 @param input_a tensor X (of type FLOATING_POINT_TYPES)

 Options:
 @option name Optional name
 @return Tensor


def sin(input_a, name: nil)
      check_allowed_types(input_a, TensorStream::Ops::FLOATING_POINT_TYPES)
      _op(:sin, input_a, name: name)
    end

def sin ( input_a , name : nil ) check_allowed_types ( input_a , TensorStream :: Ops :: FLOATING_POINT_TYPES ) _op ( :sin , input_a , name : name ) end

Computes sin of input element-wise.

 @param input_a tensor X (of type FLOATING_POINT_TYPES)

 Options:
 @option name Optional name
 @return Tensor


def sub(input_a, input_b, name: nil)
      input_a, input_b = apply_data_type_coercion(input_a, input_b)
      _op(:sub, input_a, input_b, name: name)
    end

def sub ( input_a , input_b , name : nil ) input_a , input_b = apply_data_type_coercion ( input_a , input_b ) _op ( :sub , input_a , input_b , name : name ) end

Returns x - y element-wise.

 This operation supports broadcasting

 @param input_a tensor X
 @param input_b tensor Y

 Options:
 @option name Optional name
 @return Tensor


def sum(input_a, axis_p = nil, axis: nil, name: nil, keepdims: false)
      check_allowed_types(axis_p, TensorStream::Ops::INTEGER_TYPES)
      input_a = TensorStream.convert_to_tensor(input_a)
      return input_a if input_a.shape.scalar?
      axis_p = axis_p || axis
      axis_p = cast_axis(input_a, axis_p)
      _op(:sum, input_a, axis_p, name: name, keepdims: keepdims)
    end

def sum ( input_a , axis_p = nil , axis : nil , name : nil , keepdims : false ) check_allowed_types ( axis_p , TensorStream :: Ops :: INTEGER_TYPES ) input_a = TensorStream . convert_to_tensor ( input_a ) return input_a if input_a . shape . scalar? axis_p = axis_p || axis axis_p = cast_axis ( input_a , axis_p ) _op ( :sum , input_a , axis_p , name : name , keepdims : keepdims ) end

Computes the sum of elements across dimensions of a tensor.
 Reduces input_tensor along the dimensions given in axis. Unless keepdims is true, the rank of the
 tensor is reduced by 1 for each entry in axis. If keepdims is true, the reduced dimensions are
 retained with length 1.
 If axis has no entries, all dimensions are reduced, and a tensor with a single element is returned.

 @param input_a tensor X
 @param axis_p tensor X (of type INTEGER_TYPES)

 Options:
 @option axis axis
 @option name Optional name
 @option keepdims If true, retains reduced dimensions with length 1. default (false)
 @return Tensor

Computes the sum of elements across dimensions of a tensor . Reduces input_tensor along the dimensions given in axis . Unless keepdims is true the rank of the tensor is reduced by 1 for each entry in axis . If keepdims is true the reduced dimensions are retained with length 1 . If axis has no entries all dimensions are reduced and a tensor with a single element is returned .


def tan(input_a, name: nil)
      check_allowed_types(input_a, TensorStream::Ops::FLOATING_POINT_TYPES)
      _op(:tan, input_a, name: name)
    end

def tan ( input_a , name : nil ) check_allowed_types ( input_a , TensorStream :: Ops :: FLOATING_POINT_TYPES ) _op ( :tan , input_a , name : name ) end

Computes tan of input element-wise.

 @param input_a tensor X (of type FLOATING_POINT_TYPES)

 Options:
 @option name Optional name
 @return Tensor


def tanh(input_a, name: nil)
      check_allowed_types(input_a, TensorStream::Ops::FLOATING_POINT_TYPES)
      _op(:tanh, input_a, name: name)
    end

def tanh ( input_a , name : nil ) check_allowed_types ( input_a , TensorStream :: Ops :: FLOATING_POINT_TYPES ) _op ( :tanh , input_a , name : name ) end

Computes tanh of input element-wise.

 @param input_a tensor X (of type FLOATING_POINT_TYPES)

 Options:
 @option name Optional name
 @return Tensor


def top_k(input, k = 1, sorted: true, name: nil)
      result = _op(:top_k, input, k, sorted: sorted, name: name)
      [result[0], result[1]]
    end

def top_k ( input , k = 1 , sorted : true , name : nil ) result = _op ( :top_k , input , k , sorted : sorted , name : name ) [ result [ 0 ] , result [ 1 ] ] end

Finds values and indices of the `k` largest entries for the last dimension.

 @param input 1-D or higher `Tensor` with last dimension at least `k`.
 @param k 0-D `int32` `Tensor`.  Number of top elements to look for along the last dimension (along each row for matrices)

 Options:
 @option sorted If true the resulting `k` elements will be sorted by the values in descending order. default (true)
 @option name Optional name
 @return Tensor

Finds values and indices of the k largest entries for the last dimension .


def zeros(shape, dtype: :float32, name: nil)
      _op(:zeros, shape, dtype: dtype, name: name)
    end

def zeros ( shape , dtype : :float32 , name : nil ) _op ( :zeros , shape , dtype : dtype , name : name ) end

Creates a tensor with all elements set to zero

 @param shape A 1-D integer Tensor or ruby array. The shape of the output tensor.

 Options:
 @option dtype Optional name default (:float32)
 @option name Optional name
 @return Tensor


def convert(session, checkpoint_folder, output_file)
      model_file = File.join(checkpoint_folder, "model.yaml")
      TensorStream.graph.as_default do |current_graph|
        YamlLoader.new.load_from_string(File.read(model_file))
        saver = TensorStream::Train::Saver.new
        saver.restore(session, checkpoint_folder)

        # collect all assign ops and remove them from the graph
        remove_nodes = Set.new(current_graph.nodes.values.select { |op| op.is_a?(TensorStream::Operation) && op.operation == :assign }.map { |op| op.consumers.to_a }.flatten.uniq)

        output_buffer = TensorStream::Yaml.new.get_string(current_graph) { |graph, node_key|
          node = graph.get_tensor_by_name(node_key)
          case node.operation
          when :variable_v2
            value = node.container
            options = {
              value: value,
              data_type: node.data_type,
              shape: shape_eval(value),
            }
            const_op = TensorStream::Operation.new(current_graph, inputs: [], options: options)
            const_op.name = node.name
            const_op.operation = :const
            const_op.data_type = node.data_type
            const_op.shape = TensorShape.new(shape_eval(value))

            const_op
          when :assign
            nil
          else
            remove_nodes.include?(node.name) ? nil : node
          end
        }
        File.write(output_file, output_buffer)
      end
    end

Utility class to convert variables to constants for production deployment


def device(device_name)
      Thread.current["ts_graph_#{object_id}"] ||= {}
      Thread.current["ts_graph_#{object_id}"][:default_device] ||= []
      Thread.current["ts_graph_#{object_id}"][:default_device] << device_name
      begin
        yield
      ensure
        Thread.current["ts_graph_#{object_id}"][:default_device].pop
      end
    end

def device ( device_name ) Thread . current [ "ts_graph_#{object_id}" ] ||= { } Thread . current [ "ts_graph_#{object_id}" ] [ :default_device ] ||= [ ] Thread . current [ "ts_graph_#{object_id}" ] [ :default_device ] << device_name begin yield ensure Thread . current [ "ts_graph_#{object_id}" ] [ :default_device ] . pop end end

Returns a context manager that specifies the default device to use.

Returns a context manager that specifies the default device to use .


def load_from_string(buffer)
      serialized_ops = YAML.safe_load(buffer, [Symbol], [], true)
      serialized_ops.each do |op_def|
        inputs = op_def[:inputs].map { |i| @graph.get_tensor_by_name(i) }
        options = {}

        new_var = nil
        if op_def.dig(:attrs, :container)
          new_var = Variable.new(op_def.dig(:attrs, :data_type))
          var_shape = op_def.dig(:attrs, :container, :shape)
          var_options = op_def.dig(:attrs, :container, :options)
          var_options[:name] = op_def[:name]

          new_var.prepare(var_shape.size, var_shape, TensorStream.get_variable_scope, var_options)
          options[:container] = new_var

          @graph.add_variable(new_var, var_options)
        end

        new_op = Operation.new(@graph, inputs: inputs, options: op_def[:attrs].merge(options))
        new_op.operation = op_def[:op].to_sym
        new_op.name = op_def[:name]
        new_op.shape = TensorShape.new(TensorStream::InferShape.infer_shape(new_op))
        new_op.rank = new_op.shape.rank
        new_op.data_type = new_op.set_data_type(op_def.dig(:attrs, :data_type))
        new_op.is_const = new_op.infer_const
        new_op.given_name = new_op.name
        new_var.op = new_op if new_var

        @graph.add_node(new_op)
      end
      @graph
    end

def load_from_string ( buffer ) serialized_ops = YAML . safe_load ( buffer , [ Symbol ] , [ ] , true ) serialized_ops . each do | op_def | inputs = op_def [ :inputs ] . map { | i | @graph . get_tensor_by_name ( i ) } options = { } new_var = nil if op_def . dig ( :attrs , :container ) new_var = Variable . new ( op_def . dig ( :attrs , :data_type ) ) var_shape = op_def . dig ( :attrs , :container , :shape ) var_options = op_def . dig ( :attrs , :container , :options ) var_options [ :name ] = op_def [ :name ] new_var . prepare ( var_shape . size , var_shape , TensorStream . get_variable_scope , var_options ) options [ :container ] = new_var @graph . add_variable ( new_var , var_options ) end new_op = Operation . new ( @graph , inputs : inputs , options : op_def [ :attrs ] . merge ( options ) ) new_op . operation = op_def [ :op ] . to_sym new_op . name = op_def [ :name ] new_op . shape = TensorShape . new ( TensorStream :: InferShape . infer_shape ( new_op ) ) new_op . rank = new_op . shape . rank new_op . data_type = new_op . set_data_type ( op_def . dig ( :attrs , :data_type ) ) new_op . is_const = new_op . infer_const new_op . given_name = new_op . name new_var . op = new_op if new_var @graph . add_node ( new_op ) end @graph end

Loads a model Yaml file and builds the model from it

 Args:
 buffer: String - String in Yaml format of the model

 Returns: Graph where model is restored to


def variable(value, name: nil, initializer: nil, graph: nil, dtype: nil, trainable: true)
      op = Graph.get_default_graph.add_op(:assign, nil, value)
      common_options = {
        initializer: initializer || op,
        name: name,
        graph: graph,
        dtype: dtype,
        trainable: trainable,
      }
      tensor = if value.is_a?(String)
        i_var(dtype || :string, 0, [], get_variable_scope, common_options)
      elsif value.is_a?(Integer)
        i_var(dtype || :int32, 0, [], get_variable_scope, common_options)
      elsif value.is_a?(Float)
        i_var(dtype || :float32, 0, [], get_variable_scope, common_options)
      else
        i_var(dtype || :float32, 0, nil, get_variable_scope, common_options)
      end
      op.set_input(0, tensor.op)
      Graph.get_default_graph.add_node(op)
      tensor
    end

def variable ( value , name : nil , initializer : nil , graph : nil , dtype : nil , trainable : true ) op = Graph . get_default_graph . add_op ( :assign , nil , value ) common_options = { initializer : initializer || op , name : name , graph : graph , dtype : dtype , trainable : trainable , } tensor = if value . is_a? ( String ) i_var ( dtype || :string , 0 , [ ] , get_variable_scope , common_options ) elsif value . is_a? ( Integer ) i_var ( dtype || :int32 , 0 , [ ] , get_variable_scope , common_options ) elsif value . is_a? ( Float ) i_var ( dtype || :float32 , 0 , [ ] , get_variable_scope , common_options ) else i_var ( dtype || :float32 , 0 , nil , get_variable_scope , common_options ) end op . set_input ( 0 , tensor . op ) Graph . get_default_graph . add_node ( op ) tensor end

Creates a variable
 A variable maintains state across sessions


def variable_scope(scope = nil, default_name = nil, reuse: nil, initializer: nil)
      Thread.current[:tensor_stream_variable_scope] ||= [VariableScope.new]

      # uniquenifier
      if scope.nil? && default_name
        same_names = get_variable_scope.used_names.select { |s| s.start_with?(default_name) }
        new_name = default_name
        index = 1
        while same_names.include?(new_name)
          new_name = "#{default_name}_#{index}"
          index += 1
        end
        scope = new_name
      end

      variable_scope = VariableScope.new(name: scope, reuse: reuse, initializer: initializer)
      get_variable_scope.register_name(scope || "")
      Thread.current[:tensor_stream_variable_scope] << variable_scope
      scope_name = __v_scope_name
      if block_given?
        begin
          TensorStream.get_default_graph.name_scope(scope) do
            yield(scope_name)
          end
        ensure
          Thread.current[:tensor_stream_variable_scope].pop
        end
      else
        variable_scope
      end
    end

def
variable_scope
(
scope
=
,
default_name
=
,
reuse
:
,
initializer
:
)
Thread
.
current
[
:tensor_stream_variable_scope
]
||=
[
VariableScope
.
new
]
# uniquenifier
if
scope
.
nil?
&&
default_name
same_names
=
get_variable_scope
.
used_names
.
select
{
|
s
|
s
.
start_with?
(
default_name
)
}
new_name
=
default_name
index
=
1
while
same_names
.
include?
(
new_name
)
new_name
=
"#{default_name}_#{index}"
index
+=
1
end
scope
=
new_name
end
variable_scope
=
VariableScope
.
new
(
name
:
scope
,
reuse
:
reuse
,
initializer
:
initializer
)
get_variable_scope
.
register_name
(
scope
||
""
)
Thread
.
current
[
:tensor_stream_variable_scope
]
<<
variable_scope
scope_name
=
__v_scope_name
if
block_given?
begin
TensorStream
.
get_default_graph
.
name_scope
(
scope
)
do
yield
(
scope_name
)
end
ensure
Thread
.
current
[
:tensor_stream_variable_scope
]
.
pop
end
else
variable_scope
end
end


def session(evaluator = nil, thread_pool_class: Concurrent::ImmediateExecutor, log_device_placement: false, profile_enabled: false)
      session = TensorStream::Session.new(evaluator, thread_pool_class: thread_pool_class, log_device_placement: log_device_placement, profile_enabled: profile_enabled)
      yield session if block_given?

      session
    end

def session ( evaluator = nil , thread_pool_class : Concurrent :: ImmediateExecutor , log_device_placement : false , profile_enabled : false ) session = TensorStream :: Session . new ( evaluator , thread_pool_class : thread_pool_class , log_device_placement : log_device_placement , profile_enabled : profile_enabled ) yield session if block_given? session end

Creates a session context where operations can be executed

 Args:
     evaluator: Specific evaluator to use, otherwise the best evaluator will automatically be determined

 Options:
     thread_pool_class: Class to use to manage thread pooling
     log_device_placement: Show assigned device/evalutor for each tensor op
     profile_enabled: Log performance metrics for each operation


def placeholder(dtype, shape: nil, name: nil)
      TensorStream::Placeholder.new(dtype, nil, shape, name: name)
    end

def placeholder ( dtype , shape : nil , name : nil ) TensorStream :: Placeholder . new ( dtype , nil , shape , name : name ) end

Inserts a placeholder for a tensor that will be always fed.

Inserts a placeholder for a tensor that will be always fed .


def check_if_dense(value, expected_shape = nil)
      return unless value.is_a?(Array)
      return if value.empty?

      expected_shape ||= shape_eval(value)

      s = expected_shape.shift
      raise TensorStream::ValueError, "Argument must be a dense tensor: #{value}, expected size #{s} got #{value.size}" if value.size != s

      return if expected_shape.empty?

      value.each do |item|
        check_if_dense(item, expected_shape.dup)
      end
    end

Check to make sure passed array is dense


def apply_data_type_coercion(*args)
      coerced_type = check_data_types(*args)
      args.map { |a| a.is_a?(Tensor) ? a : convert_to_tensor(a, dtype: coerced_type) }
    end

def apply_data_type_coercion ( * args ) coerced_type = check_data_types ( args ) args . map { | a | a . is_a? ( Tensor ) ? a : convert_to_tensor ( a , dtype : coerced_type ) } end

Auto cast ruby constant data types to the same
 tensor types of other operands