forked from logscape/logscape.github.io
-
Notifications
You must be signed in to change notification settings - Fork 0
/
data-flow.html
206 lines (191 loc) · 13 KB
/
data-flow.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
<!DOCTYPE html ><html lang="en"><head><title> Logscape 2.0 </title><script src="js/kiss.js"></script><link href="css/bootstrap.min.css" rel="stylesheet"/><link href="css/style.css" rel="stylesheet"/><link href="css/font-awesome-4.0.3/css/font-awesome.min.css" rel="stylesheet"/><link href="css/style-app.css" rel="stylesheet"/><link href="css/app.css" rel="stylesheet"/><link href="//vjs.zencdn.net/4.9/video-js.css" rel="stylesheet"/><script src="//vjs.zencdn.net/4.9/video.js"></script><script src="js/jquery-1.11.0.js"></script><script src="js/bootstrap.min.js"></script><script>var breadcrumbs={
"gettingstarted":"Getting Started"
,"searching":"Search"
,"kv":"Data Discovery"
,"offsets":"Using Offsets"
,"geoip":"Geo IP"
,"pattern":"Using Search Patterns"
,"video":"tutorials"
,"overlays":"Overlaying Searches"
,"starthere":""
,"ds":"Data Sources"
,"add":" Add a New Data Source"
,"multitags":" Using Multi-Tags"
,"syslog":" Connect Syslog Clients"
,"tags": " Sources and Tags"
,"intelligent_field_discovery":" Intelligent Field Discovery"
,"types":"Data Types"
,"system_fields":"System Fields"
,"deploy":"Deployment"
,"app":" Logscape Apps"
,"agents": "Agents "
,"architectures_default": " Default Deployment"
,"architectures_failover": "Failover"
,"architectures_indexstores": " Index Stores"
,"architectures_tenured":"Tenured Data"
,"architectures_zones":" Zones"
,"Backup": "Backup"
,"home":" The Home WorkspacE"
,"import":" Restoring from Backup"
,"partial": " Selective Import"
,"arch_zoning":"Distributed Topology"
,"arch_clm":"Centralized Monitoring Topology"
,"arch_indexstores":"Multiple Index Store Topology"
,"rawtcpserver": "Tcp Server"
,"syslogserer":"Syslog Server"
,"centralconfigs":"Centralized Configuration"
,"boot_properties":"Boot Properties"
,"users":"Users"
,"datagroups": "Data Groups"
,"nested": "Nested Groups"
,"ldap":"Active Directory"
};
$(window).load(function(){
function genLink(arr,idx){
if (idx<0) { return "/"; }
var l="/"
for(i=0;i<=idx;i++){
l=l + arr[i] + "-" ;
}
return l.substring(0,l.length-1) + ".html";
}
function renderCrumbs(page){
console.log("rendering breadcrumbs for page:" + page)
if (page.indexOf("-") < 0){
return ""
}
var parts=page.split("-");
var idx=0;
console.log("rendering " + parts);
var html="<a href='"+genLink(parts,-1)+"'> / Home </a>";
for(idx=0;idx<parts.length;idx++){
k=parts[idx];
html=html + "<a href="+ genLink(parts,idx) + ">";
if (breadcrumbs.hasOwnProperty(k)){
html=html + " / " + breadcrumbs[k];
}else{
html=html + " / " + k[0].toUpperCase() + k.substring(1,k.length);
}
console.log(".");
}
console.log("this far!!!");
return html
}
var elems=window.location.href.split('?')[0].split('/');
var currentPage=elems[elems.length-1].split('.')[0]
console.log("crumb>" + renderCrumbs(currentPage));
$('#widget-bc').html(renderCrumbs(currentPage));
});
</script><script>var blogFeed;
function parseRSS(url, callback) {
$.ajax({
url: document.location.protocol + '//ajax.googleapis.com/ajax/services/feed/load?v=1.0&num=10&callback=?&q=' + encodeURIComponent(url),
dataType: 'json',
success: function(data) {
callback(data.responseData.feed);
}});
}
$(window).load(function() {
parseRSS("http://blog.logscape.com/feed",function(feed){
var num = feed.entries.length;
for(i=0;i< num;i++){
var title=feed.entries[i].title;
//var link=feed.entries[i].
link=feed.entries[i].link;
$(".resource-list ul").append("<li><a href='"+link+"'>"+title+"</a> </li>")
blogFeed=feed;
}
});
});
</script><script>$(window).bind("pageshow",function(){
var $form= $("form")[0];
$form.reset();
});
</script><script>function submit_form(){
var q=$("#searchBox").val()
q=encodeURIComponent(q)
var query="https://www.google.com/?gws_rd=cr&q=site:logscape.github.io++"+q+"#q=site:logscape.github.io++"+q
var location = window.location.href;
console.log("back:" + location);
history.pushState({},"",location);
window.location.replace(query);
}</script></head><body><div style="width:100%;height:85px;background-color:#333;"><!--diva(href="/") Documentation
a(href="") Forums |
a(href="") Getting Started |
a(href="") Logscape |
--><div style="padding-left:100px;padding-top:25px"><img src="images/logscape-apps-logo.png" style="height:45px"/><div style="float:right" class="col-md-3"><form id="cse-search-box" action="" style="position:right;padding-top:5px;padding-bottom:5px" onSubmit="submit_form()"><input type="hidden" name="cx" value="partner-pub-2789521296837340:9402765321"/><input type="hidden" name="ie" value="UTF-8"/><input id="searchBox" type="text" name="q" size="31" style="margin-left:15px"/><input id="cseSubmit" type="submit" value=" " class="btn-search"/></form></div></div></div><div style="width:100%;height:40px;background-color:#EEE;box-shadow: 0 0 2px #999;"><div style="padding-left:140px;padding-top:10px;font-size:14px"><a href="http://apps.logscape.com">Apps | </a><!--a(href="http://support.liquidlabs.co.uk") Release 1.3 | --><a href="http://logscape.activeboard.com/">Forums | </a><a href="gettingstarted.html">Getting Started </a><img style="height:0;width:0" src="http://logscape.com/images/track.png?version=support"/></div></div><!--.container-fluid.navbar(style="min-height:30px;margin-bottom:5px;").row(style="height:5px")
.row
.col-md-8
.col-md-1--><div class="container-fluid"><!--.row.col-md-7
form#cse-search-box(action="",style="position:right;padding-top:5px;padding-bottom:5px",onSubmit="submit_form()")
a(href="/")
img(src="images/logo.png",style="padding-top:0px,padding-right:15px",width="150")
input(type="hidden",name="cx",value="partner-pub-2789521296837340:9402765321")
input(type="hidden",name="ie",value="UTF-8")
input#searchBox(type="text",name="q",size="31",style="margin-left:15px")
input.btn-search#cseSubmit(type="submit",value=" ")
a(href="http://apps.logscape.com") Apps |
a(href="http://support.liquidlabs.co.uk") Release 1.3 |
a(href="http://logscape.activeboard.com/") Forums |
a(href="gettingstarted.html") Getting Started
--><div class="row"><div class="col-md-1"></div><div class="col-md-2"></div><div class="col-md-5"><div id="widget-bc" padding-left:150px="padding-left:150px" class="breadcrumbs style"> </div></div></div><div class="row"><div style="width:100px" class="col-md-1"><p></p></div><div class="col-md-2 nav-padding"><ul class="nav nav-tabs nav-stacked"><li><a href="searching.html">Search</a></li><li><a href="workspaces.html">WorkSpaces</a></li><li><a href="ds.html">Data Sources</a></li><li><a href="types.html">Data Types </a></li><li><a href="deploy.html">Deployment</a></li><li><a href="users.html">Users </a></li><li><a href="alerts.html">Alerts </a></li><li><a href="technology.html">Technology</a></li><li><a href="https://twitter.com/logscape" data-show-count="false" data-size="large" class="twitter-follow-button">Follow @logscape</a><script>=!function(d,s,id){var js,fjs=d.getElementsByTagName(s)[0],p=/^http:/.test(d.location)?'http':'https';if(!d.getElementById(id)){js=d.createElement(s);js.id=id;js.src=p+'://platform.twitter.com/widgets.js';fjs.parentNode.insertBefore(js,fjs);}}(document, 'script', 'twitter-wjs');</script></li></ul><div style="padding-left:50px"><h4>Articles </h4></div><hr/><div class="resource-list"><ul style="list-style:none;padding-left:10px;background-color:#FFFEE7"></ul></div></div><div class="col-md-9 content-area"><h3>Data Flow Overview </h3><div class="panel-info"><div><object id="svgDoc" type="image/svg+xml" data="images/data_flow_diagram.svg"></object></div><div id="card-sources" class="card"><h3>Sources</h3><p> Data Sources pull in log data from various locations , from your filesystems, network storage and applications.</p><a href="ds.html"></a></div><div id="card-services" class="card"><h3>Services</h3><p>Services collect data from using application apis. Services are executed by each Forwarder or Indexer collecting OS KPIs, Database metrics or Applicatin Service API data </p><a href="http://apps.logscape.com">Learn More </a></div><div id="card-default" class="card"><h3>Default Moo</h3></div><div id="card-syslog" class="card"><h3>Syslog</h3><p> The Syslog format and protocol is supported by several applications, network devices and network disks to report system health. </p><a href="ds-syslog.html">Learn More </a></div><div id="card-filesystem" class="card"><h3>File Systems </h3><p>Logs are imported from the file system </p><a href="ds-add.html">Learn More </a></div><div id="card-indexing" class="card"><h3>Indexing </h3><p> Logscape ingests the data and indexes, optimizing for search. As the data is ingested Logscape learns the structure of the data and extracts known Key Value patterns </p></div></div><div><div><object id="ingestDoc" type="image/svg+xml" data="images/data_flow_ingest.svg"></object></div><div id="card0-default" class="card0"><p></p></div><div id="card-ingestion" class="card0"><h3>Ingestion</h3><p>When a data source is created files are ingested. This is a process where data is imported and indexed. During this process Logscape will extract common Key Value patterns and making these fields available at search. Typically json and xml data is recognised but other types of attributes are supported. </p><a href="">Learn More </a></div><div id="card-type" class="card0"><h3>Data Types </h3><p>Types are assigned to incoming data. A type is a loose schema that is applied on top of log or application data. The schema exposes fields which can be used at search time. </p><a href="datatypes.html">Learn More. </a></div><div id="card-ds" class="card0"><h3>Data Sources</h3><p>Data Sources describe the location of incoming data. Each data source in the system will have at least on 'tag' assigned to it. Think of a tag as a categorization or a label for your data. Multiple tags can be assigned to each data sources allowing a rich taxonomy to be modelled around the data. </p><a href="">Learn More </a></div></div><div><div><object id="searchDoc" type="image/svg+xml" data="images/data_flow_search.svg"></object></div></div><hr/><p>Using the search example above, we can see that the search is filtered to the datasource tagged <strong><em>iis</em></strong>.This tag points to data located in<em> C:\inetpub\logs\</em><br/>or whereever the data for IIS web access logs have been configured.</p><p>The iis web access log data has been assigned the type<strong><em>www </em></strong> The <em>www</em> Datatype extracts fields such as <ul><li>request</li><li>,httpCode</li><li>,userAgent </li><li>,host</li><li>,bytes</li><li>.... </li></ul>from the web log data. </p><p>The <em>UserAgent </em> field is in the example filters the results to consider only the user agents that contain the substring 'Safari'..<br/> The _tag field narrows the search to data defined by the<em>iis </em> tag. Any text before the '|' symbol is interpreted as a keyword filter.<br/> In this example we look for failed web requests</p><script>$(window).load(function() {
function go(url){ document.location.href=url;}
var svg=$("#svgDoc")[0];
var doc=svg.getSVGDocument()
var ingestDoc=$("#ingestDoc")[0].getSVGDocument();
function connect(doc,event,id,href){
$(id,doc).on(event,function(evt){document.location.href=href;});
}
//connect(doc,"click","#sources","/ds-add.html");
//connect(doc,"click","#search","/search.html");
//connect(doc,"datatypes","#datatypes","/datatypes.html");
function hideAll(cards){
for(i=0;i<cards.length;i++){
$(cards[i]).hide();
}
}
function flip(doc,id,targetDiv,cards){
$(id,doc).on("mouseenter",function(evt){
// hide all cards
hideAll(cards);
$(targetDiv).show();
console.log("Showing " + targetDiv);
});
}
//# Sources
var deck=["#card-indexing","#card-sources","#card-syslog","#card-filesystem","#card-services","#card-default"];
hideAll(deck)
flip(doc,"#sources","#card-sources",deck);
flip(doc,"#services","#card-services",deck);
flip(doc,"#syslog","#card-syslog",deck);
flip(doc,"#filesystem","#card-filesystem",deck);
flip(doc,"#indexing","#card-indexing",deck);
//# Ingestion
deck="#card-ingestion,#card-ds,#card-type,#card-default".split();
hideAll(deck)
flip(ingestDoc,"#ingestion","#card-ingestion",deck);
flip(ingestDoc,"#ds","#card-ds",deck);
flip(ingestDoc,"#type","#card-type",deck);
//#Searching
});
</script></div></div><!--hr.dark
.col-md-3
Logscape Copyright 2014(c) Registered Trademark --></div><script>$( document ).ready(function() {
$("#cseSubmit").on("click",function(e){
e.preventDefault();
submit_form();
});
$("#searchBox").on("keypress",function(e){
if (e.keyCode == 13) {
e.preventDefault();
submit_form()
}
});
});
</script><script>(function(g,i,a,n,t,s){g['SeeYourVisitors']=n;g[n]=g[n]||function(){
(g[n].q=g[n].q||[]).push(arguments)},g[n].l=1*new Date();t=i.createElement(a),
s=i.getElementsByTagName(a)[0];t.async=1;t.src='//seeyourvisitors2.appspot.com/gg.js';
s.parentNode.insertBefore(t,s)})(window,document,'script','gg');
gg('create', 'd1a8b082-8806-4793-936f-35f5e41b3592');
gg('track');</script><script src="js/ga.js"></script></body></html>