	sepal_length	sepal_width	petal_length	petal_width	variety
0	5.1	3.5	1.4	0.2	Setosa
1	4.9	3.0	1.4	0.2	Setosa
2	4.7	3.2	1.3	0.2	Setosa
3	4.6	3.1	1.5	0.2	Setosa
4	5.0	3.6	1.4	0.2	Setosa
...	...	...	...	...	...
145	6.7	3.0	5.2	2.3	Virginica
146	6.3	2.5	5.0	1.9	Virginica
147	6.5	3.0	5.2	2.0	Virginica
148	6.2	3.4	5.4	2.3	Virginica
149	5.9	3.0	5.1	1.8	Virginica

sepal_length

sepal_width

petal_length

petal_width

variety

5.1

3.5

1.4

0.2

Setosa

4.9

3.0

1.4

0.2

Setosa

4.7

3.2

1.3

0.2

Setosa

4.6

3.1

1.5

0.2

Setosa

5.0

3.6

1.4

0.2

Setosa

...

145

6.7

3.0

5.2

2.3

Virginica

146

6.3

2.5

5.0

1.9

Virginica

147

6.5

3.0

5.2

2.0

Virginica

148

6.2

3.4

5.4

2.3

Virginica

149

5.9

3.0

5.1

1.8

Virginica

	variety	sepal_length	sepal_width	petal_length	petal_width
0	Setosa	5.1	3.5	1.4	0.2
1	Setosa	4.9	3.0	1.4	0.2
2	Setosa	4.7	3.2	1.3	0.2
3	Setosa	4.6	3.1	1.5	0.2
4	Setosa	5.0	3.6	1.4	0.2
...	...	...	...	...	...
145	Virginica	6.7	3.0	5.2	2.3
146	Virginica	6.3	2.5	5.0	1.9
147	Virginica	6.5	3.0	5.2	2.0
148	Virginica	6.2	3.4	5.4	2.3
149	Virginica	5.9	3.0	5.1	1.8

variety

sepal_length

sepal_width

petal_length

petal_width

Setosa

5.1

3.5

1.4

0.2

Setosa

4.9

3.0

1.4

0.2

Setosa

4.7

3.2

1.3

0.2

Setosa

4.6

3.1

1.5

0.2

Setosa

5.0

3.6

1.4

0.2

...

145

Virginica

6.7

3.0

5.2

2.3

146

Virginica

6.3

2.5

5.0

1.9

147

Virginica

6.5

3.0

5.2

2.0

148

Virginica

6.2

3.4

5.4

2.3

149

Virginica

5.9

3.0

5.1

1.8

', 'vp-if-item', title, title); item.appendFormatLine('

', 'vp-if-item-header', description); item.appendFormatLine('

', 'vp-if-indicator'); - item.appendFormatLine('', 'vp-if-item-title', title); + item.appendFormatLine('', 'vp-if-item-title', title); item.appendFormatLine('

', 'vp-if-item-menu'); // LAB: img to url // item.appendFormatLine('

' diff --git a/visualpython/js/com/component/InstanceEditor.js b/visualpython/js/com/component/InstanceEditor.js index 31a9dca8..c3a030c0 100644 --- a/visualpython/js/com/component/InstanceEditor.js +++ b/visualpython/js/com/component/InstanceEditor.js @@ -3,8 +3,10 @@ define([ 'vp_base/js/com/com_Const', 'vp_base/js/com/com_String', 'vp_base/js/com/com_util', - 'vp_base/js/com/component/SuggestInput' -], function(insCss, com_Const, com_String, com_util, SuggestInput) { + 'vp_base/js/com/component/SuggestInput', + 'vp_base/data/m_library/instanceLibrary', + 'vp_base/js/com/component/LibraryComponent' +], function(insCss, com_Const, com_String, com_util, SuggestInput, instanceLibrary, LibraryComponent) { // temporary const @@ -35,12 +37,17 @@ define([ * @constructor */ class InstanceEditor { - constructor(pageThis, targetId, containerId = 'vp_wrapper', popup = false) { + constructor(pageThis, targetId, containerId = 'vp_wrapper', config = {}) { this.pageThis = pageThis; this.targetId = targetId; this.uuid = 'u' + com_util.getUUID(); this.containerId = containerId; - this.popup = popup; + this.config = { + popup: false, + showAlert: false, // show alert by modal or not + targetType: 'instance', // instance / outside + ...config + } this.state = { code: '', @@ -70,7 +77,6 @@ define([ return this.state.list; } init() { - this.reload(); } wrapSelector(selector = '') { @@ -166,6 +172,7 @@ define([ tag.appendFormatLine('

', VP_INS_PARAMETER_BOX); tag.appendFormatLine('', VP_INS_PARAMETER, 'input parameter'); + tag.appendFormatLine('', 'vp-ins-opt-button'); tag.appendLine('

'); // VP_INS_PARAMETER tag.appendLine('

'); // VP_INS_BOX END @@ -279,6 +286,13 @@ define([ } } }); + + // open option popup + $(document).on('click', this.wrapSelector('.vp-ins-opt-button:not(.disabled)'), function(event) { + if (that.optionPopup) { + that.optionPopup.open(); + } + }); } reload(callback = undefined) { var that = this; @@ -289,19 +303,106 @@ define([ return; } this.state.code = variable; + var prevVarType = that.state.type; if (variable == '') { if (!this.isFirstPage) { - this.renderFirstPage(); - this.isFirstPage = true; + // if it's outside mode + if (this.config.targetType === 'outside') { + this.isFirstPage = false; + this.renderPage(); + return; + } else { + this.renderFirstPage(); + this.isFirstPage = true; + } } } else { this.isFirstPage = false; this.renderPage(); } + var splitList = []; + if (variable != '') { + splitList = variable.split('.'); + } + var hasOption = false; + // get parameter + if (splitList && splitList.length > 0) { + var lastSplit = splitList[splitList.length - 1]; + // get target code + var methodMatch = lastSplit.match(/[a-zA-Z_]+/i); + if (methodMatch) { + var methodName = methodMatch[0]; + var targetCode = splitList.slice(0, splitList.length - 1).join('.'); + if ((prevVarType in instanceLibrary.INSTANCE_MATCHING_LIBRARY) && (methodName in instanceLibrary.INSTANCE_MATCHING_LIBRARY[prevVarType])) { + // get target library + var targetLib = instanceLibrary.INSTANCE_MATCHING_LIBRARY[prevVarType][methodName]; + var targetId = targetLib.target; + var popupState = { + config: { + name: methodName, category: 'Instance', + id: targetLib.id, + saveOnly: true, + noOutput: true + } + } + // add targetid as state if exists + if (targetId) { + popupState[targetId] = targetCode; + that.state[targetId] = targetCode; + } + if (that.optionPopup) { + that.optionPopup.remove(); + that.optionPopup = null; + } + that.optionPopup = new LibraryComponent(popupState, + { + pageThis: that, + targetSelector: that.pageThis.wrapSelector('#' + that.targetId), + + finish: function(code) { + // set parameter + let lastSplit = code?.split('.')?.pop(); + // if bracket is at the end of code + let matchList = lastSplit.match(/$.*?$$/gi); + if (matchList != null && matchList.length > 0) { + let lastBracket = matchList[matchList.length - 1]; + // remove first/last brackets + let parameter = lastBracket.substr(1, lastBracket.length - 2); + $(that.wrapSelector('.' + VP_INS_PARAMETER)).val(parameter); + } + + $(that.pageThis.wrapSelector('#' + that.targetId)).trigger({ + type: "instance_editor_replaced", + originCode: that.state.code, + newCode: code + }); + } + } + ); + hasOption = true; + } else { + that.optionPopup = null; + } + } + - var code = com_util.formatString('_vp_print(_vp_load_instance("{0}"))', variable); + if (hasOption) { + if ($(that.wrapSelector('.vp-ins-opt-button')).hasClass('disabled')) { + $(that.wrapSelector('.vp-ins-opt-button')).removeClass('disabled'); + } + } else { + if (!$(that.wrapSelector('.vp-ins-opt-button')).hasClass('disabled')) { + $(that.wrapSelector('.vp-ins-opt-button')).addClass('disabled'); + } + } + } else { + if (!$(that.wrapSelector('.vp-ins-opt-button')).hasClass('disabled')) { + $(that.wrapSelector('.vp-ins-opt-button')).addClass('disabled'); + } + } + var code = com_util.formatString('_vp_print(_vp_load_instance("{0}"))', variable); vpKernel.execute(code).then(function (resultObj) { let { result } = resultObj; var varObj = { @@ -316,6 +417,10 @@ define([ var varType = varObj.type; var varList = varObj.list; + if (varType == 'module') { + // get module name + varType = varObj.name; + } that.state.type = varType; that.state.list = varList; @@ -407,9 +512,9 @@ define([ }); // get parameter - var splitList = variable.split('.'); if (splitList && splitList.length > 0) { var lastSplit = splitList[splitList.length - 1]; + // if bracket is at the end of code var matchList = lastSplit.match(/$.*?$$/gi); if (matchList != null && matchList.length > 0) { @@ -417,13 +522,13 @@ define([ // remove first/last brackets var parameter = lastBracket.substr(1, lastBracket.length - 2); $(that.wrapSelector('.' + VP_INS_PARAMETER)).val(parameter); - $(that.wrapSelector('.' + VP_INS_PARAMETER)).show(); + $(that.wrapSelector('.' + VP_INS_PARAMETER)).prop('disabled', false); } else { $(that.wrapSelector('.' + VP_INS_PARAMETER)).val(''); - $(that.wrapSelector('.' + VP_INS_PARAMETER)).hide(); + $(that.wrapSelector('.' + VP_INS_PARAMETER)).prop('disabled', true); } } else { - $(that.wrapSelector('.' + VP_INS_PARAMETER)).hide(); + $(that.wrapSelector('.' + VP_INS_PARAMETER)).prop('disabled', true); } } @@ -434,16 +539,15 @@ define([ }).catch(function(resultObj) { let { result } = resultObj; // show alert if this is visible - if (that.pageThis.isHidden() == false) { + if (that.pageThis.isHidden() == false && that.config.showAlert == true) { com_util.renderAlertModal(result.ename + ': ' + result.evalue); } // callback if (callback) { - callback(''); + callback({}); } }); - } show() { $(this.wrapSelector()).show(); @@ -454,16 +558,5 @@ define([ } } - - - - - - - - - - - return InstanceEditor; }) \ No newline at end of file diff --git a/visualpython/js/com/component/LibraryComponent.js b/visualpython/js/com/component/LibraryComponent.js index 452cf0e0..e02c31f2 100644 --- a/visualpython/js/com/component/LibraryComponent.js +++ b/visualpython/js/com/component/LibraryComponent.js @@ -17,9 +17,9 @@ define([ __VP_CSS_LOADER__('vp_base/css/m_library/libraryComponent'), // INTEGRATION: unified version of css loader 'vp_base/js/com/component/PopupComponent', 'vp_base/js/com/com_Const', - 'vp_base/js/com/com_generator', + 'vp_base/js/com/com_generatorV2', 'vp_base/data/m_library/pandasLibrary' -], function(libHtml, libCss, PopupComponent, com_Const, com_generator, pandasLibrary) { +], function(libHtml, libCss, PopupComponent, com_Const, com_generatorV2, pandasLibrary) { /** * LibraryComponent @@ -30,8 +30,9 @@ define([ /** Write codes executed before rendering */ this.config.dataview = false; this.config.sizeLevel = 1; + this.config.helpview = true; - this.packageId = this.state.config.id; + this.packageId = this.id; // deep copy package info this.package = null; try { @@ -46,6 +47,42 @@ define([ return; } this.config.checkModules = ['pd']; + + // set docs link + if (this.package.docs === undefined) { + let docsLink = 'https://pandas.pydata.org/docs/reference/api/pandas.'; + let docsMatchObj = this.package.code.match(/\= (.+)\.(.+)\(/); + if (docsMatchObj) { + let targetType = docsMatchObj[1]; // ${i0} or pd + let method = docsMatchObj[2]; + if (targetType === 'pd') { + docsLink += method + '.html'; + } else { + docsLink += 'DataFrame.' + method + '.html'; + } + this.config.docs = docsLink; + } else { + this.config.docs = 'https://pandas.pydata.org/docs/reference/index.html'; + } + } else { + this.config.docs = this.package.docs; + } + + // set helpview content + let helpMatchObj = this.package.code.match(/\= (.+)\.(.+)\(/); + if (helpMatchObj) { + let helpContent = ''; + let targetType = helpMatchObj[1]; // ${i0} or pd + let method = helpMatchObj[2]; + if (targetType === 'pd') { + helpContent += '_vp_pd.' + method; + } else { + helpContent += '_vp_pd.' + 'DataFrame.' + method; + } + this.config.helpInfo.content = helpContent; + } else { + this.config.helpview = false; + } vpLog.display(VP_LOG_TYPE.DEVELOP, 'loading state', this.state); } @@ -131,8 +168,26 @@ define([ super.render(); // show interface - com_generator.vp_showInterfaceOnPage(this.wrapSelector(), this.package); + // com_generator.vp_showInterfaceOnPage(this.wrapSelector(), this.package); + if (this.config.noOutput && this.config.noOutput === true) { + // no allocateTo + this.package.options = this.package.options.filter(x => x.output != true); + } + com_generatorV2.vp_showInterfaceOnPage(this, this.package, this.state); + + // hide required page if no options + if ($.trim($(this.wrapSelector('#vp_inputOutputBox table tbody')).html())=='') { + $(this.wrapSelector('.vp-require-box')).hide(); + } + + // hide optional page if no options + if ($.trim($(this.wrapSelector('#vp_optionBox table tbody')).html())=='') { + $(this.wrapSelector('.vp-option-box')).hide(); + } + } + open() { + super.open(); // hide optional page if no options if ($.trim($(this.wrapSelector('#vp_optionBox table tbody')).html())=='') { $(this.wrapSelector('.vp-option-box')).hide(); @@ -140,7 +195,9 @@ define([ } generateCode() { - return com_generator.vp_codeGenerator(this.uuid, this.package); + // return com_generator.vp_codeGenerator(this.uuid, this.package); + let code = com_generatorV2.vp_codeGenerator(this, this.package, this.state); + return code; } } diff --git a/visualpython/js/com/component/ModelEditor.js b/visualpython/js/com/component/ModelEditor.js index 64541c7c..c2fab057 100644 --- a/visualpython/js/com/component/ModelEditor.js +++ b/visualpython/js/com/component/ModelEditor.js @@ -80,8 +80,8 @@ define([ code: '${model}.fit(${fit_featureData}, ${fit_targetData})', description: 'Perform modeling from features, or distance matrix.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X_train' }, - { name: 'fit_targetData', label: 'Target Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'y_train' } + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'fit_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' } ] }, 'predict': { @@ -90,7 +90,7 @@ define([ code: '${pred_allocate} = ${model}.predict(${pred_featureData})', description: 'Predict the closest target data X belongs to.', options: [ - { name: 'pred_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X_test' }, + { name: 'pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_test' }, { name: 'pred_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'pred' } ] }, @@ -100,7 +100,7 @@ define([ code: '${pred_prob_allocate} = ${model}.predict_proba(${pred_prob_featureData})', description: 'Predict class probabilities for X.', options: [ - { name: 'pred_prob_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X_test' }, + { name: 'pred_prob_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_test' }, { name: 'pred_prob_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'pred' } ] }, @@ -110,7 +110,7 @@ define([ code: '${trans_allocate} = ${model}.transform(${trans_featureData})', description: 'Apply dimensionality reduction to X.', options: [ - { name: 'trans_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] } @@ -125,7 +125,7 @@ define([ code: '${model}.fit(${fit_featureData})', description: 'Fit Encoder/Scaler to X.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' } + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' } ] }, 'fit_transform': { @@ -134,16 +134,21 @@ define([ code: '${fit_trans_allocate} = ${model}.fit_transform(${fit_trans_featureData})', description: 'Fit Encoder/Scaler to X, then transform X.', options: [ - { name: 'fit_trans_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'fit_trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] }, 'transform': { - ...defaultActions['transform'], - description: 'Transform labels to normalized encoding.' + name: 'transform', + label: 'Transform', + code: '${trans_allocate} = ${model}.transform(${trans_featureData})', + description: 'Transform labels to normalized encoding.', + options: [ + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, + { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } + ] } } - if (modelType != 'ColumnTransformer') { actions = { ...actions, @@ -153,12 +158,75 @@ define([ code: '${inverse_allocate} = ${model}.inverse_transform(${inverse_featureData})', description: 'Transform binary labels back to multi-class labels.', options: [ - { name: 'inverse_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'inverse_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'inverse_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'inv_trans' } ] } } } + if (modelType == 'LabelEncoder') { + actions = { + ...actions, + 'fit': { + name: 'fit', + label: 'Fit', + code: '${model}.fit(${fit_featureData})', + description: 'Fit Encoder/Scaler to X.', + options: [ + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', + columnSelection: 'single', returnFrameType: 'Series' } + ] + }, + 'fit_transform': { + name: 'fit_transform', + label: 'Fit and transform', + code: '${fit_trans_allocate} = ${model}.fit_transform(${fit_trans_featureData})', + description: 'Fit Encoder/Scaler to X, then transform X.', + options: [ + { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', + columnSelection: 'single', returnFrameType: 'Series' }, + { name: 'fit_trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } + ] + }, + 'transform': { + name: 'transform', + label: 'Transform', + code: '${trans_allocate} = ${model}.transform(${trans_featureData})', + description: 'Transform labels to normalized encoding.', + options: [ + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', + columnSelection: 'single', returnFrameType: 'Series' }, + { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } + ] + } + } + } + if (modelType === 'SMOTE') { + actions = { + 'fit': { + name: 'fit', + label: 'Fit', + code: '${model}.fit(${fit_featureData}, ${fit_targetData})', + description: 'Check inputs and statistics of the sampler.', + options: [ + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'fit_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' } + ] + }, + 'fit_resample': { + name: 'fit_resample', + label: 'Fit and resample', + code: '${fit_res_allocateX}, ${fit_res_allocatey} = ${model}.fit_resample(${fit_res_featureData}, ${fit_res_targetData})', + description: 'Resample the dataset.', + options: [ + { name: 'fit_res_allocateX', label: 'Allocate feature', component: ['input'], placeholder: 'New variable', value: 'X_res' }, + { name: 'fit_res_allocatey', label: 'Allocate target', component: ['input'], placeholder: 'New variable', value: 'y_res' }, + { name: 'fit_res_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'fit_res_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' } + ] + } + } + } break; case 'Regression': actions = { @@ -181,7 +249,7 @@ define([ code: '${dec_allocate} = ${model}.decision_function(${dec_featureData})', description: 'Compute the decision function of X.', options: [ - { name: 'dec_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'dec_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'dec_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable' } ] } @@ -198,7 +266,7 @@ define([ code: '${fit_pred_allocate} = ${model}.fit_predict(${fit_pred_featureData})', description: 'Fit and predict.', options: [ - { name: 'fit_pred_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'fit_pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'fit_pred_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'pred' } ] }, @@ -215,7 +283,7 @@ define([ code: '${model}.fit(${fit_featureData})', description: 'Perform clustering from features, or distance matrix.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' } + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' } ] }, 'fit_predict': { @@ -224,7 +292,7 @@ define([ code: '${fit_pred_allocate} = ${model}.fit_predict(${fit_pred_featureData})', description: 'Compute clusters from a data or distance matrix and predict labels.', options: [ - { name: 'fit_pred_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'fit_pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'fit_pred_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'pred' } ] } @@ -238,7 +306,7 @@ define([ code: '${model}.fit(${fit_featureData})', description: 'Compute clustering.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' } + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' } ] }, 'predict': { @@ -247,7 +315,7 @@ define([ code: '${pred_allocate} = ${model}.predict(${pred_featureData})', description: 'Predict the closest target data X belongs to.', options: [ - { name: 'pred_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'pred_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'pred' } ] }, @@ -257,7 +325,7 @@ define([ code: '${fit_pred_allocate} = ${model}.fit_predict(${fit_pred_featureData})', description: 'Compute cluster centers and predict cluster index for each sample.', options: [ - { name: 'fit_pred_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'fit_pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'fit_pred_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'pred' } ] } @@ -271,7 +339,7 @@ define([ code: '${fit_trans_allocate} = ${model}.fit_transform(${fit_trans_featureData})', description: 'Compute clustering and transform X to cluster-distance space.', options: [ - { name: 'fit_trans_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'fit_trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] }, @@ -281,7 +349,7 @@ define([ code: '${trans_allocate} = ${model}.transform(${trans_featureData})', description: 'Transform X to a cluster-distance space.', options: [ - { name: 'trans_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] } @@ -297,7 +365,7 @@ define([ code: '${model}.fit(${fit_featureData})', description: 'Fit X into an embedded space.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' } + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' } ] }, 'fit_transform': { @@ -306,7 +374,7 @@ define([ code: '${fit_trans_allocate} = ${model}.fit_transform(${fit_trans_featureData})', description: 'Fit X into an embedded space and return that transformed output.', options: [ - { name: 'fit_trans_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'fit_trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] } @@ -321,18 +389,18 @@ define([ code: '${model}.fit(${fit_featureData}, ${fit_targetData})', description: 'Fit the Linear Discriminant Analysis model.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, - { name: 'fit_targetData', label: 'Target Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'y' } + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, + { name: 'fit_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y' } ] }, 'fit_transform': { name: 'fit_transform', label: 'Fit and transform', - code: '${fit_trans_allocate} = ${model}.fit_transform(${fit_trans_featureData}${fit_trans_targetData})', + code: '${fit_trans_allocate} = ${model}.fit_transform(${fit_trans_featureData}, ${fit_trans_targetData})', description: 'Fit to data, then transform it.', options: [ - { name: 'fit_trans_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, - { name: 'fit_trans_targetData', label: 'Target Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'y' }, + { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, + { name: 'fit_trans_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y' }, { name: 'fit_trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] }, @@ -342,7 +410,7 @@ define([ code: '${pred_allocate} = ${model}.predict(${pred_featureData})', description: 'Predict class labels for samples in X.', options: [ - { name: 'pred_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'pred_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'pred' } ] }, @@ -352,7 +420,7 @@ define([ code: '${trans_allocate} = ${model}.transform(${trans_featureData})', description: 'Project data to maximize class separation.', options: [ - { name: 'trans_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] } @@ -366,7 +434,7 @@ define([ code: '${model}.fit(${fit_featureData})', description: 'Fit X into an embedded space.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' } + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' } ] }, 'fit_transform': { @@ -375,7 +443,7 @@ define([ code: '${fit_trans_allocate} = ${model}.fit_transform(${fit_trans_featureData})', description: 'Fit the model with X and apply the dimensionality reduction on X.', options: [ - { name: 'fit_trans_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'fit_trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] }, @@ -385,7 +453,7 @@ define([ code: '${inverse_allocate} = ${model}.inverse_transform(${inverse_featureData})', description: 'Transform data back to its original space.', options: [ - { name: 'inverse_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'inverse_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'inverse_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'inv_trans' } ] }, @@ -395,12 +463,58 @@ define([ code: '${trans_allocate} = ${model}.transform(${trans_featureData})', description: 'Apply dimensionality reduction to X.', options: [ - { name: 'trans_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] } } break; + case 'ETC': + if (modelType === 'GridSearchCV') { + actions = { + 'fit': { + name: 'fit', + label: 'Fit', + code: '${model}.fit(${fit_featureData}${fit_targetData})', + description: 'Run fit with all sets of parameters.', + options: [ + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'fit_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train', usePair: true, pairKey: 'y' } + ] + }, + 'predict': { + name: 'predict', + label: 'Predict', + code: '${pred_allocate} = ${model}.predict(${pred_featureData})', + description: 'Call predict on the estimator with the best found parameters.', + options: [ + { name: 'pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_test' }, + { name: 'pred_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'pred' } + ] + }, + 'inverse_transform': { + name: 'inverse_transform', + label: 'Inverse transform', + code: '${inverse_allocate} = ${model}.inverse_transform(${inverse_featureData})', + description: 'Call inverse_transform on the estimator with the best found params.', + options: [ + { name: 'inverse_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'inverse_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'inv_trans' } + ] + }, + 'transform': { + name: 'transform', + label: 'Transform', + code: '${trans_allocate} = ${model}.transform(${trans_featureData})', + description: 'Call transform on the estimator with the best found parameters.', + options: [ + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } + ] + } + } + } + break; } return actions; } @@ -415,8 +529,8 @@ define([ code: '${score_allocate} = ${model}.score(${score_featureData}, ${score_targetData})', description: '', options: [ - { name: 'score_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, - { name: 'score_targetData', label: 'Target Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'y' }, + { name: 'score_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'score_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, { name: 'score_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'scores' } ] }, @@ -434,15 +548,67 @@ define([ name: 'permutation_importance', label: 'Permutation importance', import: 'from sklearn.inspection import permutation_importance', - code: '${importance_allocate} = permutation_importance(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${random_state}${etc})', + code: '${importance_allocate} = vp_create_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort})', description: 'Permutation importance for feature evaluation.', options: [ - { name: 'importance_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X_train' }, - { name: 'importance_targetData', label: 'Target Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'y_train' }, - { name: 'scoring', component: ['input'], usePair: true }, - { name: 'random_state', component: ['input_number'], placeholder: '123', usePair: true }, + { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, + { name: 'scoring', component: ['option_suggest'], usePair: true, type: 'text', + options: [ + 'explained_variance', 'max_error', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_root_mean_squared_error', + 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'r2', 'neg_mean_poisson_deviance', 'neg_mean_gamma_deviance', + 'neg_mean_absolute_percentage_error', + 'accuracy', 'balanced_accuracy', 'top_k_accuracy', 'average_precision', 'neg_brier_score', + 'f1', 'f1_micro', 'f1_macro', 'f1_weighted', 'f1_samples', 'neg_log_loss', 'precision', 'recall', 'jaccard', + 'roc_auc', 'roc_auc_ovr', 'roc_auc_ovo', 'roc_auc_ovr_weighted', 'roc_auc_ovo_weighted' + ] }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, { name: 'importance_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'importances' } ] + }, + 'plot_permutation_importance': { + name: 'plot_permutation_importance', + label: 'Plot permutation importance', + import: 'from sklearn.inspection import permutation_importance', + code: 'vp_plot_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort}${top_count})', + description: 'Permutation importance for feature evaluation.', + options: [ + { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, + { name: 'scoring', component: ['option_suggest'], usePair: true, type: 'text', + options: [ + 'explained_variance', 'max_error', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_root_mean_squared_error', + 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'r2', 'neg_mean_poisson_deviance', 'neg_mean_gamma_deviance', + 'neg_mean_absolute_percentage_error', + 'accuracy', 'balanced_accuracy', 'top_k_accuracy', 'average_precision', 'neg_brier_score', + 'f1', 'f1_micro', 'f1_macro', 'f1_weighted', 'f1_samples', 'neg_log_loss', 'precision', 'recall', 'jaccard', + 'roc_auc', 'roc_auc_ovr', 'roc_auc_ovo', 'roc_auc_ovr_weighted', 'roc_auc_ovo_weighted' + ] }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, + { name: 'top_count', label: 'Top count', component: ['input_number'], min: 0, usePair: true } + ] + }, + 'feature_importances': { + name: 'feature_importances', + label: 'Feature importances', + code: "${fi_allocate} = vp_create_feature_importances(${model}, ${fi_featureData}${sort})", + description: 'Allocate feature_importances_', + options: [ + { name: 'fi_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'fi_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'df_i' }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true } + ] + }, + 'plot_feature_importances': { + name: 'plot_feature_importances', + label: 'Plot feature importances', + code: "vp_plot_feature_importances(${model}, ${fi_featureData}${sort}${top_count})", + description: 'Draw feature_importances_', + options: [ + { name: 'fi_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, + { name: 'top_count', label: 'Top count', component: ['input_number'], min: 0, usePair: true }, + ] } } switch (category) { @@ -517,6 +683,19 @@ define([ } } } + if (modelType === 'SMOTE') { + infos = { + 'get_feature_names_out': { + name: 'get_feature_names_out', + label: 'Get feature names', + code: '${feature_names_allocate} = ${model}.get_feature_names_out()', + description: 'Get output feature names for transformation.', + options: [ + { name: 'feature_names_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'features' } + ] + } + } + } infos = { ...infos, 'get_params': defaultInfos['get_params'] @@ -535,8 +714,8 @@ define([ code: '${cvs_allocate} = cross_val_score(${model}, ${cvs_featureData}, ${cvs_targetData}${scoring}${cv})', description: 'Evaluate a score by cross-validation.', options: [ - { name: 'cvs_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, - { name: 'cvs_targetData', label: 'Target Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'y' }, + { name: 'cvs_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'cvs_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y' }, { name: 'scoring', component: ['option_select'], usePair: true, type: 'text', options: [ '', @@ -548,7 +727,44 @@ define([ { name: 'cvs_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'scores' } ] }, - 'permutation_importance': defaultInfos['permutation_importance'], + 'permutation_importance': { + name: 'permutation_importance', + label: 'Permutation importance', + import: 'from sklearn.inspection import permutation_importance', + code: '${importance_allocate} = vp_create_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort})', + description: 'Permutation importance for feature evaluation.', + options: [ + { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, + { name: 'scoring', component: ['option_suggest'], usePair: true, type: 'text', + options: [ + 'explained_variance', 'max_error', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_root_mean_squared_error', + 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'r2', 'neg_mean_poisson_deviance', 'neg_mean_gamma_deviance', + 'neg_mean_absolute_percentage_error' + ] }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, + { name: 'importance_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'importances' } + ] + }, + 'plot_permutation_importance': { + name: 'plot_permutation_importance', + label: 'Plot permutation importance', + import: 'from sklearn.inspection import permutation_importance', + code: 'vp_plot_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort}${top_count})', + description: 'Permutation importance for feature evaluation.', + options: [ + { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, + { name: 'scoring', component: ['option_suggest'], usePair: true, type: 'text', + options: [ + 'explained_variance', 'max_error', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_root_mean_squared_error', + 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'r2', 'neg_mean_poisson_deviance', 'neg_mean_gamma_deviance', + 'neg_mean_absolute_percentage_error' + ] }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, + { name: 'top_count', label: 'Top count', component: ['input_number'], min: 0, usePair: true } + ] + }, 'Coefficient': { name: 'coef_', label: 'Coefficient', @@ -568,6 +784,19 @@ define([ ] } } + let svcList = [ + 'DecisionTreeRegressor', + 'RandomForestRegressor', + 'GradientBoostingRegressor', + 'XGBRegressor', 'LGBMRegressor', 'CatBoostRegressor' + ]; + if (svcList.includes(modelType)) { + infos = { + ...infos, + 'feature_importances': defaultInfos['feature_importances'], + 'plot_feature_importances': defaultInfos['plot_feature_importances'] + } + } break; case 'Classification': infos = { @@ -582,8 +811,8 @@ define([ code: '${cvs_allocate} = cross_val_score(${model}, ${cvs_featureData}, ${cvs_targetData}${scoring}${cv})', description: 'Evaluate a score by cross-validation.', options: [ - { name: 'cvs_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, - { name: 'cvs_targetData', label: 'Target Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'y' }, + { name: 'cvs_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'cvs_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y' }, { name: 'scoring', component: ['option_select'], usePair: true, type: 'text', options: [ '', @@ -595,7 +824,109 @@ define([ { name: 'cvs_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'scores' } ] }, - 'permutation_importance': defaultInfos['permutation_importance'] + 'roc_curve': { + name: 'roc_curve', + label: 'ROC Curve', + import: 'from sklearn import metrics', + code: "fpr, tpr, thresholds = metrics.roc_curve(${roc_targetData}, ${model}.predict_proba(${roc_featureData})[:, 1])\ + \nplt.plot(fpr, tpr, label='ROC Curve')\ + \nplt.xlabel('Sensitivity')\ + \nplt.ylabel('Specificity')\ + \nplt.show()", + description: '', + options: [ + { name: 'roc_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_test' }, + { name: 'roc_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_test' } + ] + }, + 'auc': { + name: 'auc', + label: 'AUC', + import: 'from sklearn import metrics', + code: 'metrics.roc_auc_score(${auc_targetData}, ${model}.predict_proba(${auc_featureData})[:, 1])', + description: '', + options: [ + { name: 'auc_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_test' }, + { name: 'auc_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_test' } + ] + }, + 'permutation_importance': { + name: 'permutation_importance', + label: 'Permutation importance', + import: 'from sklearn.inspection import permutation_importance', + code: '${importance_allocate} = vp_create_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort})', + description: 'Permutation importance for feature evaluation.', + options: [ + { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, + { name: 'scoring', component: ['option_suggest'], usePair: true, type: 'text', + options: [ + 'accuracy', 'balanced_accuracy', 'top_k_accuracy', 'average_precision', 'neg_brier_score', + 'f1', 'f1_micro', 'f1_macro', 'f1_weighted', 'f1_samples', 'neg_log_loss', 'precision', 'recall', 'jaccard', + 'roc_auc', 'roc_auc_ovr', 'roc_auc_ovo', 'roc_auc_ovr_weighted', 'roc_auc_ovo_weighted' + ] }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, + { name: 'importance_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'importances' } + ] + }, + 'plot_permutation_importance': { + name: 'plot_permutation_importance', + label: 'Plot permutation importance', + import: 'from sklearn.inspection import permutation_importance', + code: 'vp_plot_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort}${top_count})', + description: 'Permutation importance for feature evaluation.', + options: [ + { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, + { name: 'scoring', component: ['option_suggest'], usePair: true, type: 'text', + options: [ + 'accuracy', 'balanced_accuracy', 'top_k_accuracy', 'average_precision', 'neg_brier_score', + 'f1', 'f1_micro', 'f1_macro', 'f1_weighted', 'f1_samples', 'neg_log_loss', 'precision', 'recall', 'jaccard', + 'roc_auc', 'roc_auc_ovr', 'roc_auc_ovo', 'roc_auc_ovr_weighted', 'roc_auc_ovo_weighted' + ] }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, + { name: 'top_count', label: 'Top count', component: ['input_number'], min: 0, usePair: true } + ] + }, + } + + // feature importances + let clfList = [ + 'DecisionTreeClassifier', + 'RandomForestClassifier', + 'GradientBoostingClassifier', + 'XGBClassifier', + 'LGBMClassifier', + 'CatBoostClassifier', + ] + if (clfList.includes(modelType)) { + infos = { + ...infos, + 'feature_importances': defaultInfos['feature_importances'], + 'plot_feature_importances': defaultInfos['plot_feature_importances'] + } + } + + // use decision_function on ROC, AUC + let decisionFunctionTypes = [ + 'LogisticRegression', 'SVC', 'GradientBoostingClassifier' + ]; + if (decisionFunctionTypes.includes(modelType)) { + infos = { + ...infos, + 'roc_curve': { + ...infos['roc_curve'], + code: "fpr, tpr, thresholds = metrics.roc_curve(${roc_targetData}, ${model}.decision_function(${roc_featureData}))\ + \nplt.plot(fpr, tpr, label='ROC Curve')\ + \nplt.xlabel('Sensitivity')\ + \nplt.ylabel('Specificity')\ + \nplt.show()" + }, + 'auc': { + ...infos['auc'], + code: 'metrics.roc_auc_score(${auc_targetData}, ${model}.decision_function(${auc_featureData}))', + } + } } break; case 'Auto ML': @@ -611,16 +942,18 @@ define([ break; case 'Clustering': infos = { - // 'Size of clusters': { - // name: 'Size of clusters', - // code: "print(f'Size of clusters: {np.bincount(pred)}')", // FIXME: model.cluster_centers_ / use model info or hide it - // options: [] - // } + 'get_params': { + ...defaultInfos['get_params'] + } } if (modelType == 'KMeans') { infos = { ...infos, + 'score': { + ...defaultInfos['score'], + description: 'Return the mean accuracy on the given test data and labels.' + }, 'cluster_centers_': { name: 'cluster_centers', label: 'Cluster centers', @@ -642,11 +975,21 @@ define([ code: "# import\nfrom scipy.cluster.hierarchy import dendrogram, ward\n\nlinkage_array = ward(${dendro_data})\ndendrogram(linkage_array, p=3, truncate_mode='level', no_labels=True)\nplt.show()", description: 'Draw a dendrogram', options: [ - { name: 'dendro_data', label: 'Data', component: ['var_select'], var_type: ['DataFrame'] } + { name: 'dendro_data', label: 'Data', component: ['data_select'], var_type: ['DataFrame'] } ] } } } + + if (modelType == 'GaussianMixture') { + infos = { + ...infos, + 'score': { + ...defaultInfos['score'], + description: 'Compute the per-sample average log-likelihood of the given data X.' + } + } + } break; case 'Dimension Reduction': if (modelType == 'LDA') { @@ -657,8 +1000,8 @@ define([ code: '${score_allocate} = ${model}.score(${score_featureData}, ${score_targetData})', description: 'Return the average log-likelihood of all samples.', options: [ - { name: 'score_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, - { name: 'score_targetData', label: 'Target Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'y' }, + { name: 'score_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'score_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y' }, { name: 'score_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'scores' } ] } @@ -686,12 +1029,45 @@ define([ code: '${score_allocate} = ${model}.score(${score_featureData})', description: 'Return the average log-likelihood of all samples.', options: [ - { name: 'score_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'score_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'score_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'scores' } ] } } break; + case 'ETC': + if (modelType === 'GridSearchCV') { + infos = { + 'best_estimator_': { + name: 'best_estimator_', + label: 'Best estimator', + code: '${best_estimator_allocate} = ${model}.best_estimator_', + description: 'Estimator that was chosen by the search.', + options: [ + { name: 'best_estimator_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'best_estimator' } + ] + }, + 'best_score_': { + name: 'best_score_', + label: 'Best score', + code: '${best_score_allocate} = ${model}.best_score_', + description: 'Mean cross-validated score of the best_estimator.', + options: [ + { name: 'best_score_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'best_score' } + ] + }, + 'best_params_': { + name: 'best_params_', + label: 'Best params', + code: '${best_params_allocate} = ${model}.best_params_', + description: 'Parameter setting that gave the best results on the hold out data.', + options: [ + { name: 'best_params_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'best_params' } + ] + } + } + } + break; } return infos; } @@ -899,11 +1275,11 @@ define([ getCode(replaceDict={}) { let code = new com_String(); - if (this.state.config.import != undefined) { - code.appendLine(this.state.config.import); + if (this.config.import != undefined) { + code.appendLine(this.config.import); code.appendLine(); } - let modelCode = com_generator.vp_codeGenerator(this.pageThis, this.state.config, this.pageThis.state); + let modelCode = com_generator.vp_codeGenerator(this.pageThis, this.config, this.pageThis.state); Object.keys(replaceDict).forEach(key => { modelCode = modelCode.replace(key, replaceDict[key]); }); diff --git a/visualpython/js/com/component/MultiSelector.js b/visualpython/js/com/component/MultiSelector.js index 729d350b..d9d3d9fa 100644 --- a/visualpython/js/com/component/MultiSelector.js +++ b/visualpython/js/com/component/MultiSelector.js @@ -45,12 +45,19 @@ define([ //======================================================================== // [CLASS] MultiSelector //======================================================================== + /** + * MultiSelector + * Usage + * this._columnSelector = new MultiSelector(this.wrapSelector('#multi-selector-id'), + { mode: 'columns', parent: [data], selectedList: this.state.indexing, allowAdd: true } + ); + */ class MultiSelector extends Component { /** * * @param {string} frameSelector query for parent component - * @param {Object} config parent:[], selectedList=[], includeList=[] + * @param {Object} config parent:[], selectedList=[], includeList=[], excludeList=[], allowAdd=true/false */ constructor(frameSelector, config) { super(frameSelector, config, {}); @@ -62,41 +69,65 @@ define([ // configuration this.config = this.state; - var { mode, type, parent, dataList=[], selectedList=[], includeList=[], excludeList=[] } = this.config; + var { + mode, type, parent, + dataList=[], selectedList=[], includeList=[], excludeList=[], + allowAdd=false, showDescription=true, + change=null + } = this.config; this.mode = mode; // variable / columns / index / ndarray0 / ndarray1 / methods / data(given data) this.parent = parent; + this.varType = type; // for mode:variable, variable type list to search this.selectedList = selectedList; this.includeList = includeList; this.excludeList = excludeList; + this.allowAdd = allowAdd; // allow adding new item + this.showDescription = showDescription; // show description on the top of the box + + this.change = change; // function (type=('add'|'remove'), list=[]) this.dataList = dataList; // [ { value, code, type }, ... ] this.pointer = { start: -1, end: -1 }; + this.loadDataList(); + } + + render() { + ; + } + + loadDataList() { var that = this; - switch (mode) { + if (this.mode !== 'variable' && this.mode !== 'data') { + if (this.parent == null || this.parent === '' || (Array.isArray(this.parent) && this.parent.length == 0)) { + this._executeCallback([]); + return; + } + } + switch (this.mode) { case 'columns': - this._getColumnList(parent, function(dataList) { + this._getColumnList(this.parent, function(dataList) { that._executeCallback(dataList); }); break; case 'variable': - this._getVariableList(type, function(dataList) { + this._getVariableList(this.varType, function(dataList) { that._executeCallback(dataList); }); break; case 'index': - this._getRowList(parent, function(dataList) { + this._getRowList(this.parent, function(dataList) { that._executeCallback(dataList); }); break; case 'ndarray0': - this._getNdarray(parent, 0, function(dataList) { + this._getNdarray(this.parent, 0, function(dataList) { that._executeCallback(dataList); }); break; case 'ndarray1': - this._getNdarray(parent, 1, function(dataList) { + this._getNdarray(this.parent, 1, function(dataList) { that._executeCallback(dataList); }); break; @@ -106,10 +137,6 @@ define([ } } - // render() { - // ; - // } - _executeCallback(dataList) { if (this.includeList && this.includeList.length > 0) { dataList = dataList.filter(data => this.includeList.includes(data.code)); @@ -155,7 +182,8 @@ define([ ...x, value: x.label, code: x.value, - type: x.dtype + type: x.dtype, + isNumeric: x.is_numeric }; }); callback(colList); @@ -167,16 +195,17 @@ define([ vpKernel.getColumnList(parent).then(function(resultObj) { let { result } = resultObj; try { - var colList = JSON.parse(result); - colList = colList.map(function(x) { + var { list } = JSON.parse(result); + list = list.map(function(x) { return { ...x, value: x.label, code: x.value, - type: x.dtype + type: x.dtype, + isNumeric: x.is_numeric }; }); - callback(colList); + callback(list); } catch (e) { callback([]); } @@ -188,7 +217,7 @@ define([ vpKernel.getRowList(parent).then(function(resultObj) { let { result } = resultObj; try { - var rowList = JSON.parse(result); + var { list:rowList } = JSON.parse(result); rowList = rowList.map(function(x) { return { ...x, @@ -219,7 +248,8 @@ define([ ndList.push({ value: i, code: i, - type: 'int' + type: 'int', + location: i }); } callback(ndList); @@ -234,9 +264,10 @@ define([ } load() { - $(this.frameSelector).html(this.render()); + $(this.frameSelector).html(this.templateForMultiSelector()); this.bindEvent(); this.bindDraggable(); + this._bindItemClickEvent(); } getDataList() { @@ -247,7 +278,7 @@ define([ var name = $(colTags[i]).data('name'); var type = $(colTags[i]).data('type'); var code = $(colTags[i]).data('code'); - if (code) { + if (code != null) { dataList.push({ name: name, type: type, code: code}); } } @@ -255,11 +286,13 @@ define([ return dataList; } - render() { + templateForMultiSelector() { var that = this; var tag = new com_String(); - tag.appendLine('Drag-and-drop columns to right to select.'); + if (this.showDescription === true) { + tag.appendLine('Drag-and-drop columns to right to select.'); + } tag.appendFormatLine('

', APP_SELECT_CONTAINER, this.uuid); // select - left tag.appendFormatLine('

', APP_SELECT_LEFT); @@ -274,6 +307,7 @@ define([ $(this.wrapSelector()).val(value); $(this.wrapSelector()).trigger('change'); }); + vpSearchSuggest.setAutoFocus(false); vpSearchSuggest.setNormalFilter(true); tag.appendLine(vpSearchSuggest.toTagString()); tag.appendFormatLine('') @@ -305,12 +339,19 @@ define([ tag.appendFormatLine('

', APP_SELECT_RIGHT); var selectedList = this.dataList.filter(data => that.selectedList.includes(data.code)); tag.appendLine(this.renderSelectedBox(selectedList)); + if (this.allowAdd) { + // add item + tag.appendLine(''); + tag.appendLine('

'); + } tag.appendLine('

'); // APP_SELECT_RIGHT + tag.appendLine(''); tag.appendLine('

'); // APP_SELECT_CONTAINER return tag.toString(); } renderSelectionBox(dataList) { + let mode = this.mode; var tag = new com_String(); tag.appendFormatLine('

', APP_SELECT_BOX, 'left', APP_DROPPABLE, 'no-selection vp-scrollbar'); // get data and make draggable items @@ -322,29 +363,56 @@ define([ } else { info = ''; } + let iconStr = ''; + let infoStr = ''; + if (mode === 'columns') { + if (data.isNumeric === true) { + iconStr = ''; + } else { + iconStr = ''; + } + } else if (mode === 'variable') { + infoStr = ` | ${data.type}`; + } // render item box - tag.appendFormatLine('

{7}

' - , APP_SELECT_ITEM, APP_DRAGGABLE, data.location, data.value, data.type, data.code, info, data.value); + tag.appendFormat('

' + , APP_SELECT_ITEM, APP_DRAGGABLE, data.location, data.value, data.type, data.code, info); + tag.appendFormat('{0}{1}{2}', iconStr, data.value, infoStr); + tag.appendLine('

'); }); tag.appendLine('

'); // APP_SELECT_BOX return tag.toString(); } renderSelectedBox(dataList) { + let mode = this.mode; var tag = new com_String(); tag.appendFormatLine('

', APP_SELECT_BOX, 'right', APP_DROPPABLE, 'no-selection vp-scrollbar'); // get data and make draggable items dataList && dataList.forEach((data, idx) => { // for column : data.array parsing var info = com_util.safeString(data.array); - if (info) { - info = data.value + ':\n'; + if (info && info != 'undefined') { + info = data.value + ':\n' + info; } else { info = ''; } + let iconStr = ''; + let infoStr = ''; + if (mode === 'columns') { + if (data.isNumeric === true) { + iconStr = ''; + } else { + iconStr = ''; + } + } else if (mode === 'variable') { + infoStr = ` | ${data.type}`; + } // render item box - tag.appendFormatLine('

{8}

' - , APP_SELECT_ITEM, APP_DRAGGABLE, 'added', data.location, data.value, data.type, data.code, info, data.value); + tag.appendFormat('

' + , APP_SELECT_ITEM, APP_DRAGGABLE, 'added', data.location, data.value, data.type, data.code, info); + tag.appendFormat('{0}{1}{2}', iconStr, data.value, infoStr); + tag.appendLine('

'); }); tag.appendLine('

'); // APP_SELECT_BOX return tag.toString(); @@ -360,7 +428,7 @@ define([ var addedTags = $(that.wrapSelector('.' + APP_SELECT_RIGHT + ' .' + APP_SELECT_ITEM + '.added')); var addedList = []; for (var i = 0; i < addedTags.length; i++) { - var value = $(addedTags[i]).attr('data-colname'); + var value = $(addedTags[i]).attr('data-name'); addedList.push(value); } var filteredList = that.dataList.filter(x => x.value.includes(searchValue) && !addedList.includes(x.value)); @@ -372,9 +440,108 @@ define([ // draggable that.bindDraggable(); + that._bindItemClickEvent(); + }); + + // item indexing - add all + $(this.wrapSelector('.' + APP_SELECT_ADD_ALL_BTN)).on('click', function(event) { + $(that.wrapSelector('.' + APP_SELECT_BOX + '.left .' + APP_SELECT_ITEM)).appendTo( + $(that.wrapSelector('.' + APP_SELECT_BOX + '.right')) + ); + $(that.wrapSelector('.' + APP_SELECT_ITEM)).addClass('added'); + $(that.wrapSelector('.' + APP_SELECT_ITEM + '.selected')).removeClass('selected'); + that.pointer = { start: -1, end: -1 }; + + that.change && that.change('add', that.getDataList()); }); + // item indexing - add + $(this.wrapSelector('.' + APP_SELECT_ADD_BTN)).on('click', function(event) { + var selector = '.selected'; + + $(that.wrapSelector('.' + APP_SELECT_ITEM + selector)).appendTo( + $(that.wrapSelector('.' + APP_SELECT_BOX + '.right')) + ); + $(that.wrapSelector('.' + APP_SELECT_ITEM + selector)).addClass('added'); + $(that.wrapSelector('.' + APP_SELECT_ITEM + selector)).removeClass('selected'); + that.pointer = { start: -1, end: -1 }; + + that.change && that.change('add', that.getDataList()); + }); + + // item indexing - del + $(this.wrapSelector('.' + APP_SELECT_DEL_BTN)).on('click', function(event) { + var selector = '.selected'; + var targetBoxQuery = that.wrapSelector('.' + APP_SELECT_BOX + '.left'); + + var selectedTag = $(that.wrapSelector('.' + APP_SELECT_ITEM + selector)); + selectedTag.appendTo( + $(targetBoxQuery) + ); + // sort + $(targetBoxQuery + ' .' + APP_SELECT_ITEM).sort(function(a, b) { + return ($(b).data('idx')) < ($(a).data('idx')) ? 1 : -1; + }).appendTo( + $(targetBoxQuery) + ); + selectedTag.removeClass('added'); + selectedTag.removeClass('selected'); + that.pointer = { start: -1, end: -1 }; + + that.change && that.change('remove', that.getDataList()); + }); + + // item indexing - delete all + $(this.wrapSelector('.' + APP_SELECT_DEL_ALL_BTN)).on('click', function(event) { + var targetBoxQuery = that.wrapSelector('.' + APP_SELECT_BOX + '.left'); + $(that.wrapSelector('.' + APP_SELECT_BOX + '.right .' + APP_SELECT_ITEM)).appendTo( + $(targetBoxQuery) + ); + // sort + $(targetBoxQuery + ' .' + APP_SELECT_ITEM).sort(function(a, b) { + return ($(b).data('idx')) < ($(a).data('idx')) ? 1 : -1; + }).appendTo( + $(targetBoxQuery) + ); + $(that.wrapSelector('.' + APP_SELECT_ITEM)).removeClass('added'); + $(that.wrapSelector('.' + APP_SELECT_ITEM + '.selected')).removeClass('selected'); + that.pointer = { start: -1, end: -1 }; + + that.change && that.change('remove', that.getDataList()); + }); + + // add new item + $(this.wrapSelector('.vp-cs-add-item-btn')).on('click', function(event) { + let newItemName = $(that.wrapSelector('.vp-cs-add-item-name')).val(); + that._addNewItem(newItemName); + + that.change && that.change('add', that.getDataList()); + }); + // add new item (by pushing enter key) + $(this.wrapSelector('.vp-cs-add-item-name')).on('keyup', function(event) { + var keycode = event.keyCode + ? event.keyCode + : event.which; + if (keycode == 13) { // enter + let newItemName = $(this).val(); + that._addNewItem(newItemName); + + that.change && that.change('add', that.getDataList()); + } + }); + + // refresh + $(this.wrapSelector('.vp-cs-refresh')).on('click', function(event) { + that.loadDataList(); + }); + + this._bindItemClickEvent(); + } + + _bindItemClickEvent() { + let that = this; // item indexing + $(this.wrapSelector('.' + APP_SELECT_ITEM)).off('click'); $(this.wrapSelector('.' + APP_SELECT_ITEM)).on('click', function(event) { var dataIdx = $(this).attr('data-idx'); var idx = $(this).index(); @@ -430,64 +597,70 @@ define([ } }); - // item indexing - add all - $(this.wrapSelector('.' + APP_SELECT_ADD_ALL_BTN)).on('click', function(event) { - $(that.wrapSelector('.' + APP_SELECT_BOX + '.left .' + APP_SELECT_ITEM)).appendTo( - $(that.wrapSelector('.' + APP_SELECT_BOX + '.right')) - ); - $(that.wrapSelector('.' + APP_SELECT_ITEM)).addClass('added'); - $(that.wrapSelector('.' + APP_SELECT_ITEM + '.selected')).removeClass('selected'); + // item deleting (manually added item only) + $(this.wrapSelector('.vp-cs-del-item')).off('click'); + $(this.wrapSelector('.vp-cs-del-item')).on('click', function(event) { + $(this).closest('.' + APP_SELECT_ITEM).remove(); that.pointer = { start: -1, end: -1 }; - }); - - // item indexing - add - $(this.wrapSelector('.' + APP_SELECT_ADD_BTN)).on('click', function(event) { - var selector = '.selected'; - $(that.wrapSelector('.' + APP_SELECT_ITEM + selector)).appendTo( - $(that.wrapSelector('.' + APP_SELECT_BOX + '.right')) - ); - $(that.wrapSelector('.' + APP_SELECT_ITEM + selector)).addClass('added'); - $(that.wrapSelector('.' + APP_SELECT_ITEM + selector)).removeClass('selected'); - that.pointer = { start: -1, end: -1 }; + that.change && that.change('remove', that.getDataList()); }); + } - // item indexing - del - $(this.wrapSelector('.' + APP_SELECT_DEL_BTN)).on('click', function(event) { - var selector = '.selected'; - var targetBoxQuery = that.wrapSelector('.' + APP_SELECT_BOX + '.left'); - - var selectedTag = $(that.wrapSelector('.' + APP_SELECT_ITEM + selector)); - selectedTag.appendTo( - $(targetBoxQuery) - ); - // sort - $(targetBoxQuery + ' .' + APP_SELECT_ITEM).sort(function(a, b) { - return ($(b).data('idx')) < ($(a).data('idx')) ? 1 : -1; - }).appendTo( - $(targetBoxQuery) - ); - selectedTag.removeClass('added'); - selectedTag.removeClass('selected'); - that.pointer = { start: -1, end: -1 }; - }); + _addNewItem(newItemName) { + if (newItemName && newItemName !== '') { + // check if it is already exist + // - if it is already added, just select that item + // get added items + var addedTags = $(this.wrapSelector('.' + APP_SELECT_RIGHT + ' .' + APP_SELECT_ITEM + '.added')); + var addedList = []; + for (var i = 0; i < addedTags.length; i++) { + var value = $(addedTags[i]).attr('data-name'); + addedList.push(value); + } + if (addedList.includes(newItemName)) { + // just select that item and do nothing + var targetTag = $(this.wrapSelector(`.vp-cs-select-item.added[data-name="${newItemName}"]`)); + this.pointer = { start: targetTag.index(), end: -1 }; + // un-select others + $(this.wrapSelector('.' + APP_SELECT_ITEM)).removeClass('selected'); + targetTag.addClass('selected'); + return; + } + var filteredList = this.dataList.filter(x => x.value === newItemName); + if (filteredList.length > 0) { + // already exist -> move it to selected-box + var targetTag = $(this.wrapSelector(`.vp-cs-select-item[data-name="${newItemName}"]`)); + $(targetTag).appendTo( + $(this.wrapSelector('.' + APP_SELECT_BOX + '.right')) + ); + this.pointer = { start: targetTag.index(), end: -1 }; + // un-select others + $(this.wrapSelector('.' + APP_SELECT_ITEM)).removeClass('selected'); + targetTag.addClass('added'); + targetTag.addClass('selected'); + return; + } - // item indexing - delete all - $(this.wrapSelector('.' + APP_SELECT_DEL_ALL_BTN)).on('click', function(event) { - var targetBoxQuery = that.wrapSelector('.' + APP_SELECT_BOX + '.left'); - $(that.wrapSelector('.' + APP_SELECT_BOX + '.right .' + APP_SELECT_ITEM)).appendTo( - $(targetBoxQuery) + // add item + let newItemIndex = this.dataList.length; + var targetTag = $(`

+ ${newItemName} +

`); + $(targetTag).appendTo( + $(this.wrapSelector('.' + APP_SELECT_BOX + '.right')) ); - // sort - $(targetBoxQuery + ' .' + APP_SELECT_ITEM).sort(function(a, b) { - return ($(b).data('idx')) < ($(a).data('idx')) ? 1 : -1; - }).appendTo( - $(targetBoxQuery) - ); - $(that.wrapSelector('.' + APP_SELECT_ITEM)).removeClass('added'); - $(that.wrapSelector('.' + APP_SELECT_ITEM + '.selected')).removeClass('selected'); - that.pointer = { start: -1, end: -1 }; - }); + this.pointer = { start: targetTag.index(), end: -1 }; + // un-select others + $(this.wrapSelector('.' + APP_SELECT_ITEM)).removeClass('selected'); + // clear item input + $(this.wrapSelector('.vp-cs-add-item-name')).val(''); + // bind click event + this._bindItemClickEvent(); + // bind draggable + this.bindDraggable(); + } } bindDraggable() { @@ -538,6 +711,7 @@ define([ if ($(this).hasClass('right')) { // add $(dropGroup).addClass('added'); + that.change && that.change('add', that.getDataList()); } else { // del $(dropGroup).removeClass('added'); @@ -545,6 +719,7 @@ define([ $(droppedOn).find('.' + APP_SELECT_ITEM).sort(function(a, b) { return ($(b).data('idx')) < ($(a).data('idx')) ? 1 : -1; }).appendTo( $(droppedOn) ); + that.change && that.change('remove', that.getDataList()); } // remove selection $(droppableQuery).find('.selected').removeClass('selected'); diff --git a/visualpython/js/com/component/NumpyComponent.js b/visualpython/js/com/component/NumpyComponent.js index 092d83aa..d4bb217f 100644 --- a/visualpython/js/com/component/NumpyComponent.js +++ b/visualpython/js/com/component/NumpyComponent.js @@ -31,11 +31,11 @@ define([ this.config.dataview = false; this.config.sizeLevel = 1; - this.packageId = this.state.config.id; + this.packageId = this.id; // deep copy package info this.package = null; try { - let packageName = this.state.config.path.split(' - ')[2]; + let packageName = this.path.split(' - ')[2]; let findPackage = null; if (packageName == 'numpy') { findPackage = numpyLibrary.NUMPY_LIBRARIES[this.packageId]; @@ -53,6 +53,16 @@ define([ } this.config.checkModules = ['np']; + if (this.package.docs === undefined) { + if (packageName == 'numpy') { + this.config.docs = 'https://numpy.org/doc/'; + } else if (packageName == 'python') { + this.config.docs = 'https://docs.python.org/3.11/library/index.html'; + } + } else { + this.config.docs = this.package.docs; + } + vpLog.display(VP_LOG_TYPE.DEVELOP, 'loading state', this.state); } diff --git a/visualpython/js/com/component/PackageManager.js b/visualpython/js/com/component/PackageManager.js new file mode 100644 index 00000000..a808315e --- /dev/null +++ b/visualpython/js/com/component/PackageManager.js @@ -0,0 +1,479 @@ +/* + * Project Name : Visual Python + * Description : GUI-based Python code generator + * File Name : PackageManager.js + * Author : Black Logic + * Note : Component > PackageManager + * License : GNU GPLv3 with Visual Python special exception + * Date : 2023. 06. 14 + * Change Date : + */ +//============================================================================ +// [CLASS] PackageManager +//============================================================================ +define([ + __VP_TEXT_LOADER__('vp_base/html/component/packageManager.html'), // INTEGRATION: unified version of text loader + __VP_CSS_LOADER__('vp_base/css/component/packageManager'), // INTEGRATION: unified version of css loader + 'vp_base/js/com/com_util', + 'vp_base/js/com/com_Const', + 'vp_base/js/com/com_String', + 'vp_base/js/com/com_interface', + 'vp_base/js/com/component/SuggestInput', + 'vp_base/js/com/component/PopupComponent', + 'vp_base/js/com/component/FileNavigation', + 'vp_base/js/com/component/LoadingSpinner' +], function(ifHtml, ifCss, com_util, com_Const, com_String, com_interface, SuggestInput, PopupComponent, FileNavigation, LoadingSpinner) { + + /** + * PackageManager + */ + class PackageManager extends PopupComponent { + _init() { + super._init(); + /** Write codes executed before rendering */ + this.name = 'Package Manager'; + this.config.codeview = false; + this.config.dataview = false; + this.config.runButton = false; + this.config.sizeLevel = 1; + + this.state = { + selected: '', + popupType: '', + ...this.state + } + + this.packageLib = {}; + this.packageLibTemplate = { + 'numpy': { pipName: 'numpy' }, + 'pandas': { pipName: 'pandas' }, + 'pyarrow': { pipName: 'pyarrow' }, + 'matplotlib': { pipName: 'matplotlib' }, + 'seaborn': { pipName: 'seaborn' }, + 'plotly': { pipName: 'plotly' }, + 'wordcloud': { pipName: 'wordcloud' }, + 'sklearn': { pipName: 'scikit-learn' }, + 'scikit_posthocs': { pipName: 'scikit-posthocs' }, + 'scipy': { pipName: 'scipy' }, + 'statsmodels': { pipName: 'statsmodels' }, + 'factor_analyzer': { pipName: 'factor-analyzer' }, + 'pingouin': { pipName: 'pingouin' }, + 'category_encoders': { pipName: 'category_encoders' }, + 'imblearn': { pipName: 'imblearn' }, + 'xgboost': { pipName: 'xgboost' }, + 'lightgbm': { pipName: 'lightgbm' }, + 'catboost': { pipName: 'catboost' }, + 'autosklearn': { pipName: 'auto-sklearn' }, + 'tpot': { pipName: 'tpot' }, + 'pymupdf': { pipName: 'pymupdf' }, + 'sweetviz': { pipName: 'sweetviz' }, + } + + if (vpConfig.extensionType === 'lite') { + this.packageLibTemplate = { + 'numpy': { pipName: 'numpy' }, + 'pandas': { pipName: 'pandas' }, + 'matplotlib': { pipName: 'matplotlib' }, + 'seaborn': { pipName: 'seaborn' }, + 'plotly': { pipName: 'plotly' }, + 'sklearn': { pipName: 'scikit-learn' }, + 'scikit_posthocs': { pipName: 'scikit-posthocs' }, + 'scipy': { pipName: 'scipy' }, + 'statsmodels': { pipName: 'statsmodels' }, + 'factor_analyzer': { pipName: 'factor-analyzer' }, + 'category_encoders': { pipName: 'category_encoders' }, + 'imblearn': { pipName: 'imblearn' }, + 'xgboost': { pipName: 'xgboost' }, + 'lightgbm': { pipName: 'lightgbm' }, + 'catboost': { pipName: 'catboost' }, + 'autosklearn': { pipName: 'auto-sklearn' }, + 'sweetviz': { pipName: 'sweetviz' }, + } + } + } + + _unbindEvent() { + super._unbindEvent(); + $(document).off('change', this.wrapSelector('.vp-pm-search')); + } + + _bindEvent() { + super._bindEvent(); + /** Implement binding events */ + let that = this; + + // search item + $(document).on('change', this.wrapSelector('.vp-pm-search'), function(evt) { + var value = $(this).val(); + if (value != '') { + $(that.wrapSelector('.vp-pm-item')).hide(); + $(that.wrapSelector('.vp-pm-item')).filter(function() { + let key = $(this).data('key'); + return key.search(value.toLowerCase()) >= 0; + }).show(); + } else { + $(that.wrapSelector('.vp-pm-item')).show(); + } + }); + + // sort menu popup + $(this.wrapSelector('.vp-pm-sort')).on('click', function(evt) { + evt.stopPropagation(); + $(that.wrapSelector('.vp-pm-sort-menu-box')).toggle(); + }); + + // sort item + $(this.wrapSelector('.vp-pm-sort-menu-item')).on('click', function() { + var menu = $(this).data('menu'); + if (menu === 'registered') { + // sort by name + $(that.wrapSelector('.vp-pm-item')).sort(function(a, b) { + var keyA = parseInt($(a).data('seq')); + var keyB = parseInt($(b).data('seq')); + return keyA > keyB ? 1 : -1 + }).appendTo($(that.wrapSelector('.vp-pm-table'))) + } else if (menu === 'name') { + // sort by name + $(that.wrapSelector('.vp-pm-item')).sort(function(a, b) { + var keyA = $(a).data('key'); + var keyB = $(b).data('key'); + return keyA > keyB ? 1 : -1 + }).appendTo($(that.wrapSelector('.vp-pm-table'))) + } else if (menu === 'installed') { + // sort by date + $(that.wrapSelector('.vp-pm-item')).sort(function(a, b) { + var insA = $(a).data('installed'); + var insB = $(b).data('installed'); + if (insA === insB) { + var keyA = $(a).data('key'); + var keyB = $(b).data('key'); + return keyA > keyB ? 1 : -1 + } + return insA < insB ? 1 : -1 + }).appendTo($(that.wrapSelector('.vp-pm-table'))) + } else if (menu === 'uninstalled') { + $(that.wrapSelector('.vp-pm-item')).sort(function(a, b) { + var insA = $(a).data('installed'); + var insB = $(b).data('installed'); + if (insA === insB) { + var keyA = $(a).data('key'); + var keyB = $(b).data('key'); + return keyA > keyB ? 1 : -1 + } + return insA > insB ? 1 : -1 + }).appendTo($(that.wrapSelector('.vp-pm-table'))) + } + $(that.wrapSelector('.vp-pm-sort-menu-box')).hide(); + }); + + // reload package list + $(this.wrapSelector('.vp-pm-func-reload')).on('click', function() { + // reset search keyword + $(that.wrapSelector('.vp-pm-search')).val(''); + + // load package list + that.loadPackageList(); + }); + + // add package + $(this.wrapSelector('.vp-pm-add')).on('click', function() { + that.openOptionPopup('add'); + }); + } + + bindItemEvent() { + let that = this; + + // item menu click + $(this.wrapSelector('.vp-pm-item-menu-item')).off('click'); + $(this.wrapSelector('.vp-pm-item-menu-item:not(.disabled)')).on('click', function(evt) { + var menu = $(this).data('menu'); + var item = $(this).closest('.vp-pm-item'); + var key = $(item).data('key'); + if (menu === 'install') { + that.state.selected = key; + that.openOptionPopup('install'); + } else if (menu === 'uninstall') { + var pipName = that.packageLib[key].pipName; + var code = com_util.formatString("!pip uninstall -y {0}", pipName); + if (vpConfig.extensionType === 'lite') { + code = com_util.formatString("%pip uninstall {0}", pipName); + } + // DEPRECATED: no longer save to block as default + // create block and run it + // $('#vp_wrapper').trigger({ + // type: 'create_option_page', + // blockType: 'block', + // menuId: 'lgExe_code', + // menuState: { taskState: { code: code } }, + // afterAction: 'run' + // }); + com_interface.insertCell('code', code, true, 'Package Manager'); + that.loadPackageList(); + } else if (menu === 'upgrade') { + var pipName = that.packageLib[key].pipName; + var code = com_util.formatString("!pip install --upgrade {0}", pipName); + if (vpConfig.extensionType === 'lite') { + code = com_util.formatString("%pip install {0}", pipName); + } + // DEPRECATED: no longer save to block as default + // create block and run it + // $('#vp_wrapper').trigger({ + // type: 'create_option_page', + // blockType: 'block', + // menuId: 'lgExe_code', + // menuState: { taskState: { code: code } }, + // afterAction: 'run' + // }); + com_interface.insertCell('code', code, true, 'Package Manager'); + that.loadPackageList(); + } else if (menu === 'delete') { + $(item).remove(); + delete that.packageLib[key]; + vpConfig.removeData('packageList', 'vppackman').then(function() { + vpConfig.setData({ 'packageList': that.packageLib }, 'vppackman'); + that.loadPackageList(); + }); + } + evt.stopPropagation(); + }); + } + + templateForBody() { + return ifHtml; + } + + templateForAddPage() { + return `

+ Package name + + Pip name + +

`; + } + + templateForInstallPage() { + return `

+ Version selection +

+ Latest version +

+ Specified version + +

`; + } + + openOptionPopup(type) { + let that = this; + let title = ''; + let size = { width: 400, height: 250 }; + + $(this.wrapSelector('.vp-inner-popup-body')).empty(); + + this.state.popupType = type; + switch (type) { + case 'add': + title = 'Add new package to manage' + $(this.wrapSelector('.vp-inner-popup-body')).html(this.templateForAddPage()); + break; + case 'install': + title = 'Install package' + // set content + $(this.wrapSelector('.vp-inner-popup-body')).html(this.templateForInstallPage()); + + $(this.wrapSelector('.vp-inner-popup-body input[name="ver_select"]')).on('change', function() { + let checkedType = $(this).val(); + if (checkedType === 'specified') { + $(that.wrapSelector('.vp-inner-popup-version')).prop('disabled', false); + } else { + $(that.wrapSelector('.vp-inner-popup-version')).prop('disabled', true); + } + }); + break; + } + + // set size and position + $(this.wrapSelector('.vp-inner-popup-box')).css({ + width: size.width, + height: size.height, + left: 'calc(50% - ' + (size.width/2) + 'px)', + top: 'calc(50% - ' + (size.height/2) + 'px)', + }); + + // show popup box + this.openInnerPopup(title); + } + + handleInnerOk() { + switch (this.state.popupType) { + case 'add': + var packName = $(this.wrapSelector('.vp-inner-popup-package')).val(); + var pipName = $(this.wrapSelector('.vp-inner-popup-pip')).val(); + if (pipName === '') { + pipName = packName; + } + this.packageLib[packName] = { pipName: pipName }; + vpConfig.setData({ 'packageList': this.packageLib }, 'vppackman'); + + // load package list + this.loadPackageList(); + break; + case 'install': + let versionType = $(this.wrapSelector('.vp-inner-popup-body input[name="ver_select"]:checked')).val(); + var pipName = this.packageLib[this.state.selected].pipName; + var code = com_util.formatString("!pip install {0}", pipName); + if (vpConfig.extensionType === 'lite') { + code = com_util.formatString("%pip install {0}", pipName); + } + if (versionType === 'specified') { + // specified version + let version = $(this.wrapSelector('.vp-inner-popup-version')).val(); + if (version && version !== '') { + code = com_util.formatString("!pip install {0}=={1}", pipName, version); + if (vpConfig.extensionType === 'lite') { + code = com_util.formatString("%pip install {0}=={1}", pipName, version); + } + } else { + $(this.wrapSelector('.vp-inner-popup-version')).focus(); + return false; + } + } + // DEPRECATED: no longer save to block as default + // create block and run it + // $('#vp_wrapper').trigger({ + // type: 'create_option_page', + // blockType: 'block', + // menuId: 'lgExe_code', + // menuState: { taskState: { code: code } }, + // afterAction: 'run' + // }); + com_interface.insertCell('code', code, true, 'Package Manager'); + + // load package list + this.loadPackageList(); + break; + } + + this.closeInnerPopup(); + } + + render() { + super.render(); + + let that = this; + let loadingSpinner = new LoadingSpinner($(this.wrapSelector('.vp-popup-body'))); + vpConfig.getData('', 'vppackman').then(function(savedData) { + // Reset abnormal data + if (savedData == undefined || savedData.packageList === undefined) { + savedData = { packageList: JSON.parse(JSON.stringify(that.packageLibTemplate)) }; + vpConfig.setData({ savedData }, 'vppackman'); + } + + that.packageLib = { + ...savedData.packageList + }; + loadingSpinner.remove(); + // load package list + that.loadPackageList(); + }).catch(function(err) { + vpLog.display(VP_LOG_TYPE.ERROR, err); + + that.packageLib = { + ...that.packageLibTemplate + }; + loadingSpinner.remove(); + // load package list + that.loadPackageList(); + }); + } + + /** + * + * @param {String} key + * @param {Object} info installed, version, path + * @param {number} index sequence of initial package list + * @returns + */ + renderPackageItem(key, info, index) { + var item = new com_String(); + item.appendFormatLine('

', 'vp-pm-item', key, info.installed===true?'1':'0', index); + item.appendFormatLine('

', 'vp-pm-item-header', (info.path?info.path:'')); + item.appendFormatLine('{0}', key); + if (info.installed === true) { + item.appendFormatLine('{0}', info.version); + } else { + item.appendLine('Not installed'); + } + item.appendFormatLine('

', 'vp-pm-item-menu'); + // install + item.appendFormatLine('

' + , 'vp-pm-item-menu-item', 'install', 'Install'); + if (vpConfig.extensionType !== 'lite') { + // upgrade + item.appendFormatLine('

' + , 'vp-pm-item-menu-item', (info.installed===true?'':'disabled'), 'upgrade', 'Upgrade'); + // uninstall + item.appendFormatLine('

' + , 'vp-pm-item-menu-item', (info.installed===true?'':'disabled'), 'uninstall', 'Uninstall'); + } + item.appendLine('

'); // end of vp-pm-item-menu + item.appendLine('

'); // end of vp-pm-item-header + // delete button + item.appendLine(''); + item.appendLine('

'); // end of vp-pm-item + return item.toString(); + } + + generateCode() { + return ''; + } + + loadPackageList() { + var that = this; + // import importlib + // _vp_pack = importlib.import_module('numpy') + // print(_vp_pack.__version__) + + // clear table except head + $(this.wrapSelector('.vp-pm-table')).html(''); + + let packageList = Object.keys(this.packageLib); + + // set auto search + let searchInput = new SuggestInput(); + searchInput.addClass('vp-pm-search vp-input'); + searchInput.setPlaceholder("Search"); + searchInput.setSuggestList(function () { return packageList; }); + $(this.wrapSelector('.vp-pm-search')).replaceWith(searchInput.toTagString()); + + let loadingSpinner = new LoadingSpinner($(this.wrapSelector('.vp-popup-body'))); + vpKernel.getPackageList(packageList).then(function(resultObj) { + let { result } = resultObj; + let packageInfo = JSON.parse(result); + + // load code list + var innerFuncCode = new com_String(); + Object.keys(packageInfo).forEach((key, idx) => { + let info = packageInfo[key]; // installed, version, path + if (info) { + var item = that.renderPackageItem(key, info, idx); + innerFuncCode.append(item); + } + }); + $(that.wrapSelector('.vp-pm-table')).html(innerFuncCode.toString()); + + // bind item menu event + that.bindItemEvent(); + }).catch(function() { + + }).finally(function() { + loadingSpinner.remove(); + }); + } + + } + + return PackageManager; +}); \ No newline at end of file diff --git a/visualpython/js/com/component/PopupComponent.js b/visualpython/js/com/component/PopupComponent.js index 8e6cf32c..60b37260 100644 --- a/visualpython/js/com/component/PopupComponent.js +++ b/visualpython/js/com/component/PopupComponent.js @@ -15,7 +15,7 @@ // CHROME: notebook/js/codemirror-ipython (function(mod) { if (typeof exports == "object" && typeof module == "object"){ // CommonJS - if (vpExtType === 'lab') { + if (vpExtType === 'lab' || vpExtType === 'lite') { mod(require("codemirror/lib/codemirror"), require("codemirror/mode/python/python") ); @@ -62,6 +62,7 @@ define([ '../com_interface', './Component', './DataSelector', + './HelpViewer', /** codemirror */ 'codemirror/lib/codemirror', @@ -70,7 +71,7 @@ define([ 'codemirror/addon/display/autorefresh', // 'notebook/js/codemirror-ipython' // LAB: disabled to avoid error FIXME: ], function(popupComponentHtml, popupComponentCss - , com_util, com_Const, com_String, com_interface, Component, DataSelector, codemirror + , com_util, com_Const, com_String, com_interface, Component, DataSelector, HelpViewer, codemirror ) { 'use strict'; @@ -81,7 +82,7 @@ define([ * Component */ class PopupComponent extends Component { - constructor(state={ config: { id: 'popup', name: 'Popup title', path: 'path/file' }}, prop={}) { + constructor(state={ config: { id: 'popup', name: 'Popup title', path: 'path/file', category: '' }}, prop={}) { // CHROME: FIXME: #site -> .notebook-vertical // super($('#site'), state, prop); super($(vpConfig.parentSelector), state, prop); @@ -89,13 +90,16 @@ define([ _init() { this.eventTarget = '#vp_wrapper'; - this.id = this.state.config.id; - this.name = this.state.config.name; - this.path = this.state.config.path; - + var { config, ...state } = this.state; + this.state = state; + var { id, name, path, category, ...restConfig } = config; + this.id = id; + this.name = name; + this.path = path; + this.category = category; this.config = { - sizeLevel: 0, // 0: 400x400 / 1: 500x500 / 2: 600x500 / 3: 750x500 + sizeLevel: 0, // 0: 400x400 / 1: 500x500 / 2: 600x500 / 3: 750x500 / 4: executeMode: 'code', // cell execute mode // show header bar buttons installButton: false, // install button (#popupInstall) // FIXME: after creating packagemanager, deprecate it @@ -104,13 +108,27 @@ define([ // show view box codeview: true, dataview: true, + helpview: false, // show footer runButton: true, footer: true, + // run type : run / run-save / add + runType: 'run', + // position and size position: { right: 10, top: 120 }, size: { width: 400, height: 550 }, - saveOnly: false, - checkModules: [] // module aliases or function names + // additional modes and infos + saveOnly: false, // apply mode + resizable: true, + autoScroll: true, + checkModules: [], // module aliases or function names + docs: 'https://visual-python.gitbook.io/docs/getting-started/welcome-to-visual-python', + helpInfo: { + content: '', // method to show using help() ex) pd.DataFrame + useHelp: true, // custom text to show on help viewer + importCode: '' // import code for help + }, + ...restConfig }; // check BoardFrame width and set initial position of popup @@ -140,8 +158,9 @@ define([ } this.cmCodeview = null; - this.cmCodeList = []; + + this.helpViewer = undefined; } wrapSelector(selector='') { @@ -283,8 +302,31 @@ define([ } } + addCheckModules(module) { + if (this.config.checkModules.includes(module)) { + return ; + } + this.config.checkModules.push(module); + } + _bindEvent() { var that = this; + // Popup click / focus event + $(this.wrapSelector()).on('click focus', function(evt) { + // Close on blur + if ($(that.wrapSelector('.vp-popup-button')).find(evt.target).length == 0) { + if (!$(evt.target).hasClass('vp-popup-codeview-box') + && $(that.wrapSelector('.vp-popup-codeview-box')).find(evt.target).length == 0) { + that.closeView('code'); + } + } + if ($(that.wrapSelector('.vp-popup-button')).find(evt.target).length == 0) { + if (!$(evt.target).hasClass('vp-popup-dataview-box') + && $(that.wrapSelector('.vp-popup-dataview-box')).find(evt.target).length == 0) { + that.closeView('data'); + } + } + }); // Close popup event $(this.wrapSelector('.vp-popup-close')).on('click', function(evt) { if (that.getTaskType() === 'task') { @@ -302,10 +344,8 @@ define([ }); // Toggle operation (minimize) $(this.wrapSelector('.vp-popup-toggle')).on('click', function(evt) { - $(that.eventTarget).trigger({ - type: 'close_option_page', - component: that - }); + evt.stopPropagation(); + that.hide(); }); // Maximize operation $(this.wrapSelector('.vp-popup-maximize')).on('click', function(evt) { @@ -346,6 +386,9 @@ define([ // add install codes var codes = that.generateInstallCode(); codes && codes.forEach(code => { + if (vpConfig.extensionType === 'lite') { + code = code.replace('!', '%'); + } com_interface.insertCell('code', code, true, that.getSigText()); }); }); @@ -385,6 +428,11 @@ define([ that._saveSingleState($(this)[0]); }); + // click input box with selection + $(document).on('focus', this.wrapSelector('input'), function() { + $(this).select(); + }); + // Click buttons $(this.wrapSelector('.vp-popup-button')).on('click', function(evt) { var btnType = $(this).data('type'); @@ -405,6 +453,14 @@ define([ } evt.stopPropagation(); break; + case 'help': + if ($(that.wrapSelector('.vp-popup-dataview-box')).is(':hidden')) { + that.openView('help'); + } else { + that.closeView('help'); + } + evt.stopPropagation(); + break; case 'cancel': if (that.getTaskType() === 'task') { $(that.eventTarget).trigger({ @@ -420,33 +476,75 @@ define([ } break; case 'run': - let result = that.run(); + var result = that.run(); if (result) { - that.save(); + // that.save(); + that.close(); } break; case 'show-detail': $(that.wrapSelector('.vp-popup-run-detailbox')).show(); + // set run button + vpConfig.getData('runType', 'vpcfg').then(function(data) { + vpLog.display(VP_LOG_TYPE.DEVELOP, 'Runtype get data', data); + that._setDefaultRunType(data); + }); evt.stopPropagation(); break; case 'save': that.save(); break; + case 'run-save': + var result = that.run(); + if (result) { + that.save(); + } + break; + case 'add': + var result = that.run(false); + if (result) { + // that.save(); + that.close(); + } + break; } }); + // Click detail radio + $(this.wrapSelector('.vp-popup-run-type')).on('click', function(evt) { + let runType = $(that.wrapSelector('.vp-popup-run-type[name=runType]:checked')).val(); + // save as vpConfig + vpConfig.setData({runType: runType}, 'vpcfg'); + that._setDefaultRunType(runType); + }); // Click detail buttons - $(this.wrapSelector('.vp-popup-detail-button')).on('click', function(evt) { + $(this.wrapSelector('.vp-popup-detail-action-button')).on('click', function(evt) { var btnType = $(this).data('type'); + vpConfig.setData({runType: btnType}, 'vpcfg'); + that._setDefaultRunType(btnType); // run, run-save, add switch(btnType) { - case 'apply': - that.save(); + // case 'apply': + // that.save(); + // break; + case 'run': + var result = that.run(); + if (result) { + // that.save(); + that.close(); + } break; - case 'add': - let result = that.run(false); + case 'run-save': + var result = that.run(); if (result) { that.save(); } break; + case 'add': + var result = that.run(false); + if (result) { + // that.save(); + that.close(); + } + break; } }); // Close event for inner popup @@ -466,6 +564,14 @@ define([ } }); + // blur on code, dataview + $(this.wrapSelector('.vp-popup-codeview-box')).on('hide', function() { + that.closeView('code'); + }); + $(this.wrapSelector('.vp-popup-dataview-box')).on('hide', function() { + that.closeView('data'); + }); + // focus on data selector input $(this.wrapSelector('.vp-data-selector')).on('focus', function(evt) { @@ -479,12 +585,13 @@ define([ _unbindEvent() { $(document).off('change', this.wrapSelector('.vp-state')); + $(document).off('focus', this.wrapSelector('input')); } _bindDraggable() { var that = this; let containment = 'body'; - if (vpConfig.extensionType === 'lab') { + if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { containment = '#main'; } $(this.wrapSelector()).draggable({ @@ -530,7 +637,12 @@ define([ template() { this.$pageDom = $(popupComponentHtml.replaceAll('${vp_base}', com_Const.BASE_PATH)); // set title - this.$pageDom.find('.vp-popup-title').text(this.name); + // this.$pageDom.find('.vp-popup-title').text(this.category + ' > ' + this.name); + if (this.category && this.category !== '') { + this.$pageDom.find('.vp-popup-title').html(`${this.name}`); + } else { + this.$pageDom.find('.vp-popup-title').html(`${this.name}`); + } // set body let bodyTemplate = this.templateForBody(); // CHROME: check url keyword and replace it @@ -556,10 +668,14 @@ define([ let { installButton, importButton, packageButton, - codeview, dataview, runButton, footer, - sizeLevel, position + codeview, dataview, helpview, runButton, footer, + sizeLevel, position, autoScroll, docs } = this.config; + // apply link to docs + if (docs) { + $(this.wrapSelector('.vp-popup-docs')).prop('href', docs); + } // import & package manager button hide/show if (!installButton) { // FIXME: Deprecated after creating package manager $(this.wrapSelector('#popupInstall')).hide(); @@ -588,6 +704,9 @@ define([ if (!dataview) { $(this.wrapSelector('.vp-popup-button[data-type="data"]')).hide(); } + if (!helpview) { + $(this.wrapSelector('.vp-popup-button[data-type="help"]')).hide(); + } // run button if (!runButton) { @@ -601,7 +720,15 @@ define([ $(this.wrapSelector('.vp-popup-body')).css({ 'height': 'calc(100% - 30px)' }) + } else { + // set run button + let that = this; + vpConfig.getData('runType', 'vpcfg').then(function(data) { + vpLog.display(VP_LOG_TYPE.DEVELOP, 'Runtype get data', data); + that._setDefaultRunType(data); + }); } + // popup-frame size switch (sizeLevel) { @@ -614,6 +741,16 @@ define([ case 3: this.config.size = { width: 760, height: 550 }; break; + case 4: + this.config.size = { width: 840, height: 550 }; + break; + case 5: + this.config.size = { width: 1064, height: 550 }; + break; + } + + if (autoScroll) { + $(this.wrapSelector('.vp-popup-body')).addClass('vp-scrollbar'); } // set detailed size @@ -631,7 +768,9 @@ define([ } this._bindDraggable(); - this._bindResizable(); + if (this.config.resizable) { + this._bindResizable(); + } } templateForInnerPopup() { @@ -691,7 +830,37 @@ define([ } loadState() { - /** Implementation needed */ + vpLog.display(VP_LOG_TYPE.DEVELOP, this.state); + + let that = this; + Object.keys(this.state).forEach(key => { + if (key && key !== '' && key !== 'config') { + let tag = $(that.wrapSelector('#' + key) + ', ' + that.wrapSelector('input[name="' + key + '"]')); + let tagName = $(tag).prop('tagName'); + let savedValue = that.state[key]; + switch(tagName) { + case 'INPUT': + let inputType = $(tag).prop('type'); + if (inputType === 'text' || inputType === 'number' || inputType === 'hidden') { + $(tag).val(savedValue); + break; + } + if (inputType === 'checkbox') { + $(tag).prop('checked', savedValue); + break; + } + if (inputType === 'radio') { + $(tag).filter(`[value="${savedValue}"]`).prop('checked', true); + } + break; + case 'TEXTAREA': + case 'SELECT': + default: + $(tag).val(savedValue); + break; + } + } + }); } saveState() { @@ -703,9 +872,27 @@ define([ vpLog.display(VP_LOG_TYPE.DEVELOP, 'savedState', that.state); } - _saveSingleState(tag) { + _setDefaultRunType(runType) { + if (runType == undefined || runType == null || runType === '') { + runType = 'run'; // default = run + } + this.config.runType = runType; + $(this.wrapSelector(`.vp-popup-run-type[value="${runType}"]`)).prop('checked', true); + $(this.wrapSelector('.vp-popup-run-button')).attr('data-type', runType); + let runTitle = 'Run code'; // when runType is 'run' + switch (runType) { + case 'run-save': + runTitle = 'Save to block & Run code'; + break; + case 'add': + runTitle = 'Add code to cell'; + break; + } + $(this.wrapSelector('.vp-popup-run-button')).prop('title', runTitle); + } + + _getTagValue(tag) { let id = tag.id; - let customKey = $(tag).data('key'); let tagName = $(tag).prop('tagName'); // returns with UpperCase let newValue = ''; switch(tagName) { @@ -713,6 +900,15 @@ define([ let inputType = $(tag).prop('type'); if (inputType == 'checkbox') { newValue = $(tag).prop('checked'); + } else if (inputType == 'radio') { + let radioGroup = $(tag).prop('name'); + let checked = $(tag).prop('checked'); + if (checked === true) { + id = radioGroup; + newValue = $(tag).val(); + } else { + return ; + } } else { // inputType == 'text' || inputType == 'number' || inputType == 'hidden' || inputType == 'color' || inputType == 'range' newValue = $(tag).val(); @@ -727,12 +923,21 @@ define([ } break; } + return newValue; + } + + _saveSingleState(tag) { + let id = tag.id; + let customKey = $(tag).data('key'); + let newValue = this._getTagValue(tag); // if custom key is available, use it if (customKey && customKey != '') { // allow custom key until level 2 let customKeys = customKey.split('.'); - if (customKeys.length == 2) { + if (customKeys.length === 3) { + this.state[customKeys[0]][customKeys[1]][customKeys[2]] = newValue; + } else if (customKeys.length === 2) { this.state[customKeys[0]][customKeys[1]] = newValue; } else { this.state[customKey] = newValue; @@ -767,7 +972,7 @@ define([ */ checkRequiredOption() { let requiredFilled = true; - let requiredTags = $(this.wrapSelector('input[required=true]') + ',' + this.wrapSelector('input[required=required]')); + let requiredTags = $(this.wrapSelector('input[required=true]:visible') + ',' + this.wrapSelector('input[required=required]:visible')); vpLog.display(VP_LOG_TYPE.DEVELOP, 'checkRequiredOption', this, requiredTags); @@ -783,9 +988,23 @@ define([ } } + if (requiredFilled) { + requiredFilled = this.checkBeforeRun(); + } + return requiredFilled; } + /** + * Check options and do some operation before run + * @returns {boolean} check options and returns true or false (false will stop code running) + */ + checkBeforeRun() { + /** Implementation needed */ + + return true; + } + checkAndRunModules(execute=true, background=false) { let sigText = this.getSigText(); @@ -799,9 +1018,9 @@ define([ let checkedList = JSON.parse(result); let executeList = []; checkedList && checkedList.forEach((mod, idx) => { - if (mod == false) { + if (mod === false) { let modInfo = vpConfig.getModuleCode(checkModules[idx]); - if (modInfo) { + if (modInfo && modInfo?.code !== '') { executeList.push(modInfo.code); } } @@ -844,35 +1063,62 @@ define([ return true; } + _beforeOpen() { + /** Implementation needed - Generated on clicking Install Package button */ + } + /** * Open popup * - show popup * - focus popup * - bind codemirror */ - open() { + open(targetFrame=undefined) { vpLog.display(VP_LOG_TYPE.DEVELOP, 'open popup', this); this.loadState(); - this.show(); - - // set popup position if its top-left side is outside of view - let pos = $(this.wrapSelector()).position(); - if (pos) { - if (pos.top < 0) { - $(this.wrapSelector()).css({ top: 0 }); - } - if (pos.left < 0) { - $(this.wrapSelector()).css({ left: 0 }); + this._beforeOpen(); + if (targetFrame !== undefined) { + // hide popup frame + $(this.wrapSelector('.vp-popup-header')).hide(); + $(this.wrapSelector('.vp-popup-footer')).hide(); + // set width and height to 100% + $(this.wrapSelector()).css({ + width: '100%', + height: '100%', + 'min-height': 'unset', + display: 'block', + position: 'initial', + border: '0px' + }); + $(this.wrapSelector('.vp-popup-body')).css({ + padding: '0px' + }); + // show on targetFrame + $(this.wrapSelector()).appendTo($(targetFrame)); + } else { + this.show(); + + // set popup position if its top-left side is outside of view + let pos = $(this.wrapSelector()).position(); + if (pos) { + if (pos.top < 0) { + $(this.wrapSelector()).css({ top: 0 }); + } + if (pos.left < 0) { + $(this.wrapSelector()).css({ left: 0 }); + } } } this._bindCodemirror(); - $(this.eventTarget).trigger({ - type: 'focus_option_page', - component: this - }); + if (targetFrame == undefined) { + $(this.eventTarget).trigger({ + type: 'focus_option_page', + component: this + }); + } } setSaveOnlyMode() { @@ -889,10 +1135,22 @@ define([ close() { vpLog.display(VP_LOG_TYPE.DEVELOP, 'close popup', this); this.saveState(); - this.hide(); + if (this.getTaskType() === 'task') { + $(this.eventTarget).trigger({ + type: 'remove_option_page', + component: this + }); + } else { + // if it's block, just hide it + this.hide(); + } } save() { + if (this.prop.finish && typeof this.prop.finish == 'function') { + var code = this.generateCode(); + this.prop.finish(code, this.state); + } $(this.eventTarget).trigger({ type: 'apply_option_page', blockType: 'block', @@ -902,7 +1160,9 @@ define([ remove() { vpLog.display(VP_LOG_TYPE.DEVELOP, 'remove popup', this); + this._unbindResizable(); this._unbindEvent(); + this.closeHelpView(); $(this.wrapSelector()).remove(); } @@ -919,6 +1179,10 @@ define([ blur() { $(this.wrapSelector()).removeClass('vp-focused'); + // blur on its block + if (this.taskItem) { + this.taskItem.blurItem(); + } } show() { @@ -926,11 +1190,28 @@ define([ $(this.wrapSelector()).show(); } + showInstallButton() { + $(this.wrapSelector('#popupInstall')).show(); + } + + showImportButton() { + $(this.wrapSelector('#popupImport')).show(); + } + hide() { + this.closeHelpView(); this.taskItem && this.taskItem.blurItem(); $(this.wrapSelector()).hide(); } + hideInstallButton() { + $(this.wrapSelector('#popupInstall')).hide(); + } + + hideImportButton() { + $(this.wrapSelector('#popupImport')).hide(); + } + isHidden() { return !$(this.wrapSelector()).is(':visible'); } @@ -957,7 +1238,7 @@ define([ * @param {*} viewType code / data */ openView(viewType) { - if (viewType == 'code') { + if (viewType === 'code') { this.saveState(); var code = this.generateCode(); let codeText = ''; @@ -973,19 +1254,72 @@ define([ setTimeout(function() { that.cmCodeview.refresh(); }, 1); - $(this.wrapSelector('.vp-popup-dataview-box')).hide(); - } else { + this.closeView('data'); + $(this.wrapSelector('.vp-popup-codeview-box')).show(); + } else if (viewType === 'data') { this.renderDataView(); - $(this.wrapSelector('.vp-popup-codeview-box')).hide(); + this.closeView('code'); + $(this.wrapSelector('.vp-popup-dataview-box')).show(); + } else if (viewType === 'help') { + this.closeView('code'); + this.closeView('data'); + this.openHelpView(); } - - $(this.wrapSelector('.vp-popup-'+viewType+'view-box')).show(); } closeView(viewType) { + if (viewType === 'code') { + // reset codeview + if (this.cmCodeview) { + this.cmCodeview.setValue(''); + this.cmCodeview.save(); + + var that = this; + setTimeout(function() { + that.cmCodeview.refresh(); + }, 1); + } + } else if (viewType === 'data') { + // reset dataview + $(this.wrapSelector('.vp-popup-dataview-box')).html(''); + } + // hide view $(this.wrapSelector('.vp-popup-'+viewType+'view-box')).hide(); } + /** + * Open HelpViewer + * - only one helpviewer at once + */ + openHelpView() { + this.closeHelpView(); + this.helpViewer = new HelpViewer(); + this.helpViewer.open(this.config.helpInfo.content, this.config.helpInfo.useHelp, this.config.helpInfo.importCode); + } + + /** + * Close helpViewer + */ + closeHelpView() { + if (this.helpViewer !== undefined) { + this.helpViewer.remove(); + } + } + + /** + * Set HelpViewer content + * @param {string} content + * @param {boolean} useHelp + * @param {string} importCode + */ + setHelpContent(content, useHelp=true, importCode='') { + this.config.helpInfo = { + content: content, + useHelp: useHelp, + importCode: importCode + }; + } + /** * Open inner popup box */ @@ -994,7 +1328,7 @@ define([ $(this.wrapSelector('.vp-inner-popup-box')).show(); // focus on first input - $(this.wrapSelector('.vp-inner-popup-box input:not(:disabled):visible:first')).focus(); + $(this.wrapSelector('.vp-inner-popup-box input:not(:readonly):not(:disabled):visible:first')).focus(); // disable Jupyter key com_interface.disableOtherShortcut(); } diff --git a/visualpython/js/com/component/SuggestInput.js b/visualpython/js/com/component/SuggestInput.js index 4cd434e4..2c7fa96e 100644 --- a/visualpython/js/com/component/SuggestInput.js +++ b/visualpython/js/com/component/SuggestInput.js @@ -17,6 +17,7 @@ define([ this._placeholder = "Select variable"; this._compID = ""; this._additionalClass = ""; + this._autoFocus = true; this._normalFilter = true; this._suggestList = new Array(); this._selectEvent = undefined; @@ -48,6 +49,13 @@ define([ setComponentID(compID = "") { this._compID = compID; } + /** + * set auto focus on enter + * @param {boolean} autoFocus + */ + setAutoFocus(autoFocus = true) { + this._autoFocus = autoFocus; + } /** * normal filter usage * @param {String} normalFilter @@ -56,7 +64,7 @@ define([ this._normalFilter = normalFilter; } /** - * suggest list + * suggest list or object list [{ label, value } ...] * @param {Array or Function} suggestList */ setSuggestList(suggestList = new Array()) { @@ -109,7 +117,7 @@ define([ // make attributes var attributes = Object.keys(this._attributes).map(key => key + '="' + this._attributes[key] + '"').join(" "); - sbTagString.appendFormatLine(``, + sbTagString.appendFormatLine(``, that.uuid, 'suggest-input-uninit', that._additionalClass, that._compID == "" ? "" : com_util.formatString("id='{0}'", that._compID), that._placeholder, that._value, attributes); $(document).on(com_util.formatString("focus.init-{0}", that.uuid), com_util.formatString(".{0}.{1}", that.uuid, 'suggest-input-uninit'), function () { @@ -118,7 +126,7 @@ define([ $(com_util.formatString(".{0}", that.uuid)).removeClass('suggest-input-uninit').addClass('suggest-input'); $(com_util.formatString(".{0}", that.uuid)).autocomplete({ - autoFocus: true, + autoFocus: that._autoFocus, minLength: minLength, source: function (req, res) { var srcList = typeof that._suggestList == "function" ? that._suggestList() : that._suggestList; @@ -160,6 +168,8 @@ define([ $(this).select(); $(com_util.formatString(".{0}", that.uuid)).autocomplete('search', $(com_util.formatString(".{0}", that.uuid)).val()); }).autocomplete('instance')._renderItem = function(ul, item) { + $(ul).addClass('vp-scrollbar-vertical'); + $(ul).css('max-height', '250px'); if (item.dtype != undefined) { return $('

').attr('data-value', item.value) .append(`

${item.label} | ${item.dtype}

`) diff --git a/visualpython/js/com/component/VarSelector.js b/visualpython/js/com/component/VarSelector.js index b1acb104..7305ea0c 100644 --- a/visualpython/js/com/component/VarSelector.js +++ b/visualpython/js/com/component/VarSelector.js @@ -239,7 +239,7 @@ define([ vpKernel.getColumnList(varName).then(function(resultObj) { try { let { result, type, msg } = resultObj; - var varResult = JSON.parse(result); + var { list } = JSON.parse(result); let newTag = new com_String(); newTag.appendFormatLine(' Use PyArrow')) + ); + } else { + $(this.wrapSelector(prefix + '#vp_inputOutputBox table tbody')).prepend( + $('').append($(`File Type`)) + .append($(' ')) + ); + } + // prepend file type selector - $(this.wrapSelector(prefix + '#vp_inputOutputBox table tbody')).prepend( - $('').append($(`File Type`)) - .append($('')) - ); var fileTypeList = Object.keys(fileTypeObj); fileTypeList.forEach(type => { $(this.wrapSelector(prefix + '#fileType')).append( @@ -294,7 +434,7 @@ define([ // prepend user option let hasAllocateTo = $(this.wrapSelector(prefix + '#o0')).length > 0; if (hasAllocateTo) { - $(this.wrapSelector(prefix + '#o0')).closest('tr').before( + $(this.wrapSelector(prefix + '#o0')).closest('tr').after( $('').append($(`User Option`)) .append($('')) ) @@ -304,9 +444,9 @@ define([ .append($('')) ) } - + $(this.wrapSelector(prefix + '#fileType')).val(selectedType); - + // add file navigation button if (pageType == 'Write') { if (selectedType == 'json') { @@ -314,7 +454,7 @@ define([ com_util.formatString('

' , 'vp-file-browser-button') ); - } else if (selectedType == 'pickle') { + } else if (selectedType == 'pickle' || selectedType == 'parquet') { $(prefix + '#path').parent().html( com_util.formatString('

' , 'vp-file-browser-button') @@ -333,6 +473,18 @@ define([ , 'vp-file-browser-button') ); } + + if (pageType === 'Read' && selectedType === 'spss') { + // show install button + this.showInstallButton(); + // show install note below File type selection + $(` + NOTE: + pyreadstat package is required to read spss file. + `).insertAfter($(this.wrapSelector('#fileType')).closest('tr')); + } else { + this.hideInstallButton(); + } // encoding suggest input $(this.wrapSelector('#encoding')).replaceWith(function() { @@ -345,8 +497,18 @@ define([ suggestInput.setPlaceholder('encoding option'); return suggestInput.toTagString(); }); - - + + // separator suggest input + $(this.wrapSelector('#sep')).replaceWith(function() { + // separator list : + var sepList = [',', '|', '\\t', '\\n', ':', ';', '-', '_', '&', '/', '\\']; + var suggestInput = new SuggestInput(); + suggestInput.setComponentID('sep'); + suggestInput.addClass('vp-input vp-state'); + suggestInput.setSuggestList(function() { return sepList; }); + suggestInput.setPlaceholder('Input separator'); + return suggestInput.toTagString(); + }); } render() { @@ -368,6 +530,10 @@ define([ } + generateInstallCode() { + return [ '!pip install pyreadstat' ]; + } + generateCode() { var pageType = $(this.wrapSelector('#vp_fileioType')).val(); var sbCode = new com_String; @@ -383,26 +549,28 @@ define([ if (pageType == 'Sample') { // sample csv code - var result = pdGen.vp_codeGenerator(this.uuid + ' #vp_fileSample', { ...this.fileState[pageType] }, userOption.toString()); + // var result = pdGen.vp_codeGenerator(this.uuid + ' #vp_fileSample', { ...this.fileState[pageType] }, userOption.toString()); + var result = pdGen.vp_codeGenerator(this, { ...this.fileState[pageType] }, this.state, userOption.toString(), parent='#vp_fileSample'); sbCode.append(result); } else if (pageType == 'Read') { var thisPkg = JSON.parse(JSON.stringify(this.fileState[pageType].package)); - thisPkg.input.push({ + thisPkg.options.push({ name: 'fileType', type: 'var' }); - var result = pdGen.vp_codeGenerator(this.uuid + ' #vp_fileRead', thisPkg, userOption.toString()); + // var result = pdGen.vp_codeGenerator(this.uuid + ' #vp_fileRead', thisPkg, userOption.toString()); + var result = pdGen.vp_codeGenerator(this, thisPkg, this.state, userOption.toString(), parent='#vp_fileRead'); sbCode.append(result); } else if (pageType == 'Write') { var thisPkg = JSON.parse(JSON.stringify(this.fileState[pageType].package)); - thisPkg.input.push({ + thisPkg.options.push({ name: 'fileType', type: 'var' }); - var result = pdGen.vp_codeGenerator(this.uuid + ' #vp_fileWrite', thisPkg, userOption.toString()); + // var result = pdGen.vp_codeGenerator(this.uuid + ' #vp_fileWrite', thisPkg, userOption.toString()); + var result = pdGen.vp_codeGenerator(this, thisPkg, this.state, userOption.toString(), parent='#vp_fileWrite'); sbCode.append(result); } - return sbCode.toString(); } diff --git a/visualpython/js/m_apps/Frame.js b/visualpython/js/m_apps/Frame.js index ce4cdba0..4e91083d 100644 --- a/visualpython/js/m_apps/Frame.js +++ b/visualpython/js/m_apps/Frame.js @@ -13,16 +13,17 @@ // [CLASS] Frame //============================================================================ define([ - __VP_TEXT_LOADER__('vp_base/html/m_apps/frame.html'), // INTEGRATION: unified version of text loader - __VP_CSS_LOADER__('vp_base/css/m_apps/frame'), // INTEGRATION: unified version of css loader + __VP_TEXT_LOADER__('vp_base/html/m_apps/frame.html'), // INTEGRATION: unified version of text loader + __VP_CSS_LOADER__('vp_base/css/m_apps/frame'), // INTEGRATION: unified version of css loader 'vp_base/js/com/com_Const', 'vp_base/js/com/com_String', 'vp_base/js/com/com_util', 'vp_base/js/com/component/PopupComponent', 'vp_base/js/com/component/SuggestInput', - 'vp_base/js/com/component/VarSelector', + 'vp_base/js/com/component/DataSelector', + 'vp_base/js/com/component/MultiSelector', 'vp_base/js/m_apps/Subset' -], function(frameHtml, frameCss, com_Const, com_String, com_util, PopupComponent, SuggestInput, VarSelector, Subset) { +], function(frameHtml, frameCss, com_Const, com_String, com_util, PopupComponent, SuggestInput, DataSelector, MultiSelector, Subset) { /** * Frame @@ -30,16 +31,20 @@ define([ class Frame extends PopupComponent { _init() { super._init(); - this.config.sizeLevel = 3; + this.config.sizeLevel = 5; this.config.checkModules = ['pd']; // state this.state = { originObj: '', tempObj: '_vp', - returnObj: '_vp', + returnObj: 'vp_df', inplace: false, + menu: '', + menuItem: '', + columnLevel: 1, columnList: [], + indexLevel: 1, indexList: [], selected: [], axis: FRAME_AXIS.NONE, @@ -57,12 +62,77 @@ define([ } // numpy.dtype or python type - this.astypeList = [ - 'datetime64', - 'int', 'int32', 'int64', - 'float', 'float64', - 'object', 'category', - 'bool', 'str' + this.astypeList = [ + 'object', 'int64', 'float64', 'bool', + 'datetime64[ns]', 'timedelta[ns]', 'category' + ]; + + // { + // id: 'id', + // label: 'menu label', + // child: [ + // { + // id: 'id', label: 'label', code: 'code', + // axis: 'col/row', single_select: true/false, + // numeric_only: true/false + // } + // ] + // } + this.menuList = [ + { + id: 'edit', + label: 'Edit', + child: [ + { id: 'add_col', label: 'Add column', selection: FRAME_SELECT_TYPE.NONE, menuType: FRAME_EDIT_TYPE.ADD_COL }, + { id: 'add_row', label: 'Add row', selection: FRAME_SELECT_TYPE.NONE, menuType: FRAME_EDIT_TYPE.ADD_ROW }, + { id: 'delete', label: 'Delete', selection: FRAME_SELECT_TYPE.MULTI, menuType: FRAME_EDIT_TYPE.DROP }, + { id: 'rename', label: 'Rename', selection: FRAME_SELECT_TYPE.NONE, menuType: FRAME_EDIT_TYPE.RENAME }, + { id: 'as_type', label: 'As type', selection: FRAME_SELECT_TYPE.NONE, axis: FRAME_AXIS.COLUMN, menuType: FRAME_EDIT_TYPE.AS_TYPE }, + { id: 'to_datetime', label: 'To Datetime', selection: FRAME_SELECT_TYPE.SINGLE, axis: FRAME_AXIS.COLUMN, menuType: FRAME_EDIT_TYPE.TO_DATETIME }, + { id: 'replace', label: 'Replace', selection: FRAME_SELECT_TYPE.SINGLE, axis: FRAME_AXIS.COLUMN, menuType: FRAME_EDIT_TYPE.REPLACE }, + { id: 'discretize', label: 'Discretize', selection: FRAME_SELECT_TYPE.SINGLE, axis: FRAME_AXIS.COLUMN, numeric_only: true, menuType: FRAME_EDIT_TYPE.DISCRETIZE } + ] + }, + { + id: 'transform', + label: 'Transform', + child: [ + { id: 'set_index', label: 'Set index', axis: FRAME_AXIS.COLUMN, selection: FRAME_SELECT_TYPE.MULTI, menuType: FRAME_EDIT_TYPE.SET_IDX }, + { id: 'reset_index', label: 'Reset index', selection: FRAME_SELECT_TYPE.NONE, menuType: FRAME_EDIT_TYPE.RESET_IDX }, + { id: 'data_shift', label: 'Data shift', axis: FRAME_AXIS.COLUMN, selection: FRAME_SELECT_TYPE.NONE, menuType: FRAME_EDIT_TYPE.DATA_SHIFT } + ] + }, + { + id: 'sort', + label: 'Sort', + axis: FRAME_AXIS.COLUMN, + child: [ + { id: 'sort_index', label: 'Sort index', selection: FRAME_SELECT_TYPE.NONE, menuType: FRAME_EDIT_TYPE.SORT_INDEX }, + { id: 'sort_values', label: 'Sort values', axis: FRAME_AXIS.COLUMN, selection: FRAME_SELECT_TYPE.MULTI, menuType: FRAME_EDIT_TYPE.SORT_VALUES }, + ] + }, + { + id: 'encoding', + label: 'Encoding', + axis: FRAME_AXIS.COLUMN, + selection: FRAME_SELECT_TYPE.MULTI, + child: [ + { id: 'label_encoding', label: 'Label encoding', axis: FRAME_AXIS.COLUMN, selection: FRAME_SELECT_TYPE.MULTI, menuType: FRAME_EDIT_TYPE.LABEL_ENCODING }, + { id: 'one_hot_encoding', label: 'Onehot encoding', axis: FRAME_AXIS.COLUMN, selection: FRAME_SELECT_TYPE.MULTI, menuType: FRAME_EDIT_TYPE.ONE_HOT_ENCODING }, + ] + }, + { + id: 'data_cleaning', + label: 'Data cleaning', + axis: FRAME_AXIS.COLUMN, + child: [ + { id: 'fillna', label: 'Fill NA', axis: FRAME_AXIS.COLUMN, selection: FRAME_SELECT_TYPE.NONE, menuType: FRAME_EDIT_TYPE.FILL_NA }, + { id: 'dropna', label: 'Drop NA', axis: FRAME_AXIS.COLUMN, selection: FRAME_SELECT_TYPE.NONE, menuType: FRAME_EDIT_TYPE.DROP_NA }, + { id: 'fill_outlier', label: 'Fill outlier', axis: FRAME_AXIS.COLUMN, selection: FRAME_SELECT_TYPE.MULTI, menuType: FRAME_EDIT_TYPE.FILL_OUT }, + { id: 'drop_outlier', label: 'Drop outlier', axis: FRAME_AXIS.COLUMN, selection: FRAME_SELECT_TYPE.MULTI, menuType: FRAME_EDIT_TYPE.DROP_OUT }, + { id: 'drop_duplicates', label: 'Drop duplicates', axis: FRAME_AXIS.COLUMN, selection: FRAME_SELECT_TYPE.NONE, menuType: FRAME_EDIT_TYPE.DROP_DUP }, + ] + }, ]; // Add/Replace - subset @@ -71,7 +141,7 @@ define([ this.loading = false; - this._addCodemirror('previewCode', this.wrapSelector('#vp_fePreviewCode'), 'readonly'); + // this._addCodemirror('previewCode', this.wrapSelector('#vp_fePreviewCode'), 'readonly'); } loadState() { @@ -108,7 +178,7 @@ define([ // initialize state values that.state.originObj = origin; that.state.tempObj = '_vp'; - that.state.returnObj = that.state.tempObj; + that.state.returnObj = 'vp_df'; if (that.state.inplace === true) { that.state.returnObj = origin; } @@ -160,7 +230,7 @@ define([ // check/uncheck inplace $(this.wrapSelector('#inplace')).on('change', function() { let checked = $(this).prop('checked'); - let returnVariable = '_vp'; + let returnVariable = 'vp_df'; if (checked === true) { returnVariable = that.state.originObj; } @@ -172,19 +242,29 @@ define([ that.setPreview(that.getCurrentCode()); }); - // menu on column + // menu on column (Deprecated on v2.3.6 - Temporarily Show on v.2.4.4) $(document).on('contextmenu', this.wrapSelector('.' + VP_FE_TABLE + ' .' + VP_FE_TABLE_COLUMN), function(event) { event.preventDefault(); + + var idx = $(that.wrapSelector('.' + VP_FE_TABLE_COLUMN)).index(this); // 1 ~ n var hasSelected = $(this).hasClass('selected'); $(that.wrapSelector('.' + VP_FE_TABLE + ' .' + VP_FE_TABLE_ROW)).removeClass('selected'); // select col/idx if (!hasSelected) { + $(that.wrapSelector('.' + VP_FE_TABLE + ' .' + VP_FE_TABLE_COLUMN)).removeClass('selected'); + $(that.wrapSelector('.' + VP_FE_TABLE + ' .' + VP_FE_TABLE_COLUMN_GROUP)).removeClass('selected'); + $(this).addClass('selected'); + that.state.selection = { start: idx, end: -1 }; var newAxis = $(this).data('axis'); that.state.axis = newAxis; } + // select its group + $(that.wrapSelector(`.${VP_FE_TABLE} th[data-label="${$(this).data('parent')}"]`)).addClass('selected'); that.loadInfo(); + // load toolbar + that.renderToolbar(); // show menu var thisPos = $(this).position(); @@ -192,19 +272,24 @@ define([ that.showMenu(thisPos.left, thisPos.top + thisRect.height); }); - // menu on row + // menu on row (Deprecated on v2.3.6 - Temporarily Show on v.2.4.4) $(document).on('contextmenu', this.wrapSelector('.' + VP_FE_TABLE + ' .' + VP_FE_TABLE_ROW), function(event) { event.preventDefault(); + var idx = $(that.wrapSelector('.' + VP_FE_TABLE_ROW)).index(this); // 0 ~ n var hasSelected = $(this).hasClass('selected'); $(that.wrapSelector('.' + VP_FE_TABLE + ' .' + VP_FE_TABLE_COLUMN)).removeClass('selected'); // select col/idx if (!hasSelected) { + $(that.wrapSelector('.' + VP_FE_TABLE + ' .' + VP_FE_TABLE_ROW)).removeClass('selected'); $(this).addClass('selected'); + that.state.selection = { start: idx, end: -1 }; var newAxis = $(this).data('axis'); that.state.axis = newAxis; } that.loadInfo(); + // load toolbar + that.renderToolbar(); // show menu var thisPos = $(this).position(); @@ -214,6 +299,165 @@ define([ that.showMenu(tblPos.left + thisRect.width, tblPos.top + thisPos.top - scrollTop); }); + // un-select every selection and menu box + $(document).on('click', this.wrapSelector('.vp-fe-table'), function(evt) { + evt.stopPropagation(); + var target = evt.target; + if ($('.' + VP_FE_TABLE + ' .' + VP_FE_TABLE_COLUMN).find(target).length == 0 + && !$(target).hasClass('.' + VP_FE_TABLE + ' .' + VP_FE_TABLE_COLUMN) ) { + $(that.wrapSelector('.' + VP_FE_TABLE + ' .' + VP_FE_TABLE_ROW)).removeClass('selected'); + $(that.wrapSelector('.' + VP_FE_TABLE + ' .' + VP_FE_TABLE_COLUMN)).removeClass('selected'); + $(that.wrapSelector('.' + VP_FE_TABLE + ' .' + VP_FE_TABLE_COLUMN_GROUP)).removeClass('selected'); + $(that.wrapSelector('.vp-fe-menu-box')).hide(); + + // reset selected columns/indexes + that.state.axis = FRAME_AXIS.NONE; + that.state.selected = []; + // load toolbar + that.renderToolbar(); + } + }); + + // select column group + $(document).on('click', this.wrapSelector('.' + VP_FE_TABLE + ' .' + VP_FE_TABLE_COLUMN_GROUP), function(evt) { + evt.stopPropagation(); + + let hasSelected = $(this).hasClass('selected'); + let colLabel = $(this).data('label'); + let firstIdx = $(that.wrapSelector(`.${VP_FE_TABLE} th[data-parent="${colLabel}"]:first`)).index(); + let lastIdx = $(that.wrapSelector(`.${VP_FE_TABLE} th[data-parent="${colLabel}"]:last`)).index(); + if (firstIdx === lastIdx) { + lastIdx = -1; + } + + $(that.wrapSelector('.' + VP_FE_TABLE + ' .' + VP_FE_TABLE_ROW)).removeClass('selected'); + + if (vpEvent.keyManager.keyCheck.ctrlKey) { + if (!hasSelected) { + that.state.selection = { start: firstIdx, end: -1 }; + $(this).addClass('selected'); + $(that.wrapSelector(`.${VP_FE_TABLE} th[data-parent="${colLabel}"]`)).addClass('selected'); + var newAxis = $(this).data('axis'); + that.state.axis = newAxis; + } else { + $(this).removeClass('selected'); + $(that.wrapSelector(`.${VP_FE_TABLE} th[data-parent="${colLabel}"]`)).removeClass('selected'); + } + } else if (vpEvent.keyManager.keyCheck.shiftKey) { + var axis = that.state.axis; + var startIdx = that.state.selection.start; + if (axis != FRAME_AXIS.COLUMN) { + startIdx = -1; + } + + if (startIdx == -1) { + // no selection + that.state.selection = { start: firstIdx, end: -1 }; + } else if (startIdx > firstIdx) { + // add selection from idx to startIdx + var tags = $(that.wrapSelector('.' + VP_FE_TABLE_COLUMN)); + let parentSet = new Set(); + for (var i = firstIdx - 1; i <= startIdx; i++) { + $(tags[i]).addClass('selected'); + parentSet.add($(tags[i]).data('parent')); + } + parentSet.forEach(parentKey => { + let length = $(that.wrapSelector(`.${VP_FE_TABLE} th[data-parent="${parentKey}"]`)).length; + let selectedLength = $(that.wrapSelector(`.${VP_FE_TABLE} th.selected[data-parent="${parentKey}"]`)).length; + if (length === selectedLength) { + $(that.wrapSelector(`.${VP_FE_TABLE} th[data-label="${parentKey}"]`)).addClass('selected'); + } else { + $(that.wrapSelector(`.${VP_FE_TABLE} th[data-label="${parentKey}"]`)).removeClass('selected'); + } + }); + that.state.selection = { start: startIdx, end: firstIdx }; + } else if (startIdx <= firstIdx) { + // add selection from startIdx to idx + var tags = $(that.wrapSelector('.' + VP_FE_TABLE_COLUMN)); + let parentSet = new Set(); + for (var i = startIdx; i < lastIdx; i++) { + $(tags[i]).addClass('selected'); + parentSet.add($(tags[i]).data('parent')); + } + parentSet.forEach(parentKey => { + let length = $(that.wrapSelector(`.${VP_FE_TABLE} th[data-parent="${parentKey}"]`)).length; + let selectedLength = $(that.wrapSelector(`.${VP_FE_TABLE} th.selected[data-parent="${parentKey}"]`)).length; + if (length === selectedLength) { + $(that.wrapSelector(`.${VP_FE_TABLE} th[data-label="${parentKey}"]`)).addClass('selected'); + } else { + $(that.wrapSelector(`.${VP_FE_TABLE} th[data-label="${parentKey}"]`)).removeClass('selected'); + } + }); + that.state.selection = { start: startIdx, end: lastIdx }; + } + } else { + $(that.wrapSelector('.' + VP_FE_TABLE + ' .' + VP_FE_TABLE_COLUMN)).removeClass('selected'); + $(that.wrapSelector('.' + VP_FE_TABLE + ' .' + VP_FE_TABLE_COLUMN_GROUP)).removeClass('selected'); + + $(this).addClass('selected'); + $(that.wrapSelector(`.${VP_FE_TABLE} th[data-parent="${colLabel}"]`)).addClass('selected'); + that.state.selection = { start: firstIdx, end: lastIdx }; + var newAxis = $(this).data('axis'); + that.state.axis = newAxis; + } + that.loadInfo(); + // load toolbar + that.renderToolbar(); + }); + + /** + * Drag and drop selection TODO: release for later version v2.3.8 + */ + // drag start column + // $(document).on('mousedown', this.wrapSelector('.' + VP_FE_TABLE + ' .' + VP_FE_TABLE_COLUMN), function(evt) { + // var idx = $(that.wrapSelector('.' + VP_FE_TABLE_COLUMN)).index(this); // 1 ~ n + // that.state.axis = FRAME_AXIS.COLUMN; + // that.state.selection = { start: idx, end: -1 }; + // that.isMouseDown = true; + // console.log('down'); + + // $(document).on('mouseover', that.wrapSelector('.' + VP_FE_TABLE + ' .' + VP_FE_TABLE_COLUMN), function(evt) { + // evt.stopPropagation(); + // var idx = $(that.wrapSelector('.' + VP_FE_TABLE_COLUMN)).index(this); // 1 ~ n + // var axis = that.state.axis; + // var startIdx = that.state.selection.start; + // if (that.isMouseDown === true) { + // if (axis === FRAME_AXIS.ROW) { + // startIdx = -1; + // } + + // if (startIdx == -1) { + // // no selection + // that.state.selection = { start: idx, end: -1 }; + // } else if (startIdx > idx) { + // // add selection from idx to startIdx + // var tags = $(that.wrapSelector('.' + VP_FE_TABLE_COLUMN)); + // for (var i = idx; i <= startIdx; i++) { + // $(tags[i]).addClass('selected'); + // } + // that.state.selection = { start: startIdx, end: idx }; + // } else if (startIdx <= idx) { + // // add selection from startIdx to idx + // var tags = $(that.wrapSelector('.' + VP_FE_TABLE_COLUMN)); + // for (var i = startIdx; i <= idx; i++) { + // $(tags[i]).addClass('selected'); + // } + // that.state.selection = { start: startIdx, end: idx }; + // } + + // that.loadInfo(); + // // load toolbar + // that.renderToolbar(); + // } + // }); + // }); + + // $(document).on('mouseup', this.wrapSelector('.' + VP_FE_TABLE + ' .' + VP_FE_TABLE_COLUMN), function(evt) { + // that.isMouseDown = false; + // console.log('up'); + // $(document).off('mouseover', that.wrapSelector('.' + VP_FE_TABLE + ' .' + VP_FE_TABLE_COLUMN)); + // }); + // select column $(document).on('click', this.wrapSelector('.' + VP_FE_TABLE + ' .' + VP_FE_TABLE_COLUMN), function(evt) { evt.stopPropagation(); @@ -236,7 +480,7 @@ define([ } else if (vpEvent.keyManager.keyCheck.shiftKey) { var axis = that.state.axis; var startIdx = that.state.selection.start; - if (axis != FRAME_AXIS.COLUMN) { + if (axis === FRAME_AXIS.ROW) { startIdx = -1; } @@ -260,16 +504,19 @@ define([ } } else { $(that.wrapSelector('.' + VP_FE_TABLE + ' .' + VP_FE_TABLE_COLUMN)).removeClass('selected'); - if (!hasSelected) { - $(this).addClass('selected'); - that.state.selection = { start: idx, end: -1 }; - var newAxis = $(this).data('axis'); - that.state.axis = newAxis; - } else { - $(this).removeClass('selected'); - } + $(that.wrapSelector('.' + VP_FE_TABLE + ' .' + VP_FE_TABLE_COLUMN_GROUP)).removeClass('selected'); + + $(this).addClass('selected'); + that.state.selection = { start: idx, end: -1 }; + var newAxis = $(this).data('axis'); + that.state.axis = newAxis; } + // select its group + $(that.wrapSelector(`.${VP_FE_TABLE} th[data-label="${$(this).data('parent')}"]`)).addClass('selected'); + that.loadInfo(); + // load toolbar + that.renderToolbar(); }); // select row @@ -328,6 +575,8 @@ define([ } } that.loadInfo(); + // load toolbar + that.renderToolbar(); }); // add column @@ -348,8 +597,9 @@ define([ that.loadCode(that.getTypeCode(FRAME_EDIT_TYPE.SHOW), true); }); - // click toolbar item - $(document).on('click', this.wrapSelector('.vp-fe-toolbar-item'), function() { + // click toolbar item (Deprecated on v2.3.6 - Temporarily Show on v.2.4.4) + $(document).on('click', this.wrapSelector('.vp-fe-toolbar-item'), function(evt) { + evt.stopPropagation(); var itemType = $(this).data('type'); switch (parseInt(itemType)) { case FRAME_EDIT_TYPE.ADD_COL: @@ -360,7 +610,7 @@ define([ }); // click menu item - $(document).on('click', this.wrapSelector('.' + VP_FE_MENU_ITEM), function(event) { + $(document).on('click', this.wrapSelector('.' + VP_FE_MENU_ITEM + ':not(.disabled)'), function(event) { event.stopPropagation(); var editType = $(this).data('type'); switch (parseInt(editType)) { @@ -369,13 +619,19 @@ define([ case FRAME_EDIT_TYPE.RENAME: case FRAME_EDIT_TYPE.REPLACE: case FRAME_EDIT_TYPE.AS_TYPE: - that.openInputPopup(editType); - break; + case FRAME_EDIT_TYPE.TO_DATETIME: + case FRAME_EDIT_TYPE.DISCRETIZE: + case FRAME_EDIT_TYPE.DATA_SHIFT: + case FRAME_EDIT_TYPE.SORT_INDEX: + case FRAME_EDIT_TYPE.SORT_VALUES: + case FRAME_EDIT_TYPE.FILL_NA: + case FRAME_EDIT_TYPE.DROP_NA: + case FRAME_EDIT_TYPE.DROP_DUP: + case FRAME_EDIT_TYPE.FILL_OUT: case FRAME_EDIT_TYPE.DROP_OUT: - that.config.checkModules = ['pd', 'np', 'vp_drop_outlier']; - that.checkAndRunModules(true).then(function() { - that.loadCode(that.getTypeCode(editType)); - }); + case FRAME_EDIT_TYPE.DROP: + // open inner popup + that.openInputPopup(editType); break; default: that.loadCode(that.getTypeCode(editType)); @@ -405,10 +661,10 @@ define([ var df = $(this).val(); vpKernel.getColumnList(df).then(function(resultObj) { let { result } = resultObj; - var colList = JSON.parse(result); + var { list } = JSON.parse(result); var tag = new com_String(); tag.appendFormatLine('', 'vp-inner-popup-var2col'); - colList && colList.forEach(col => { + list && list.forEach(col => { tag.appendFormatLine('' , col.value, col.label, col.label); }); @@ -450,14 +706,47 @@ define([ var type = parseInt(this.state.popup.type); var content = this.getPopupContent(type); // required data check - if (type == FRAME_EDIT_TYPE.ADD_COL) { - if (content.name === '') { + if (type === FRAME_EDIT_TYPE.ADD_COL + || type === FRAME_EDIT_TYPE.ADD_ROW) { + if (content.name === '' || content.name === "''") { + $(this.wrapSelector('.vp-inner-popup-input0')).focus(); return; } + } else if (type === FRAME_EDIT_TYPE.REPLACE) { + ; + } else if (type === FRAME_EDIT_TYPE.FILL_NA) { + if (content.method === 'value' && content.value === '') { + $(this.wrapSelector('.vp-inner-popup-value')).focus(); + return; + } + } else if (type === FRAME_EDIT_TYPE.FILL_OUT) { + if (content.filltype === 'value' && content.fillvalue === '') { + $(this.wrapSelector('.vp-inner-popup-fillvalue')).focus(); + return; + } + } else if (type === FRAME_EDIT_TYPE.TO_DATETIME) { + if (content.newcol === '') { + $(this.wrapSelector('.vp-inner-popup-todt-new-col')).focus(); + } } - var code = this.loadCode(this.getTypeCode(this.state.popup.type, content)); - if (code == '') { - return; + // run check modules for outliers and load codes + if (type === FRAME_EDIT_TYPE.FILL_OUT) { + this.config.checkModules = ['pd', 'np', 'vp_fill_outlier']; + let that = this; + this.checkAndRunModules(true).then(function() { + that.loadCode(that.getTypeCode(that.state.popup.type, content)); + }); + } else if (type === FRAME_EDIT_TYPE.DROP_OUT) { + this.config.checkModules = ['pd', 'np', 'vp_drop_outlier']; + let that = this; + this.checkAndRunModules(true).then(function() { + that.loadCode(that.getTypeCode(that.state.popup.type, content)); + }); + } else { + var code = this.loadCode(this.getTypeCode(this.state.popup.type, content)); + if (code == '') { + return; + } } this.closeInnerPopup(); } @@ -470,6 +759,7 @@ define([ $(document).off('click', this.wrapSelector('.vp-fe-df-refresh')); $(document).off('click', this.wrapSelector('.' + VP_FE_INFO)); $(document).off('change', this.wrapSelector('#vp_feReturn')); + $(document).off('click', this.wrapSelector('.vp-popup-body')); $(document).off('contextmenu', this.wrapSelector('.' + VP_FE_TABLE + ' .' + VP_FE_TABLE_COLUMN)); $(document).off('contextmenu', this.wrapSelector('.' + VP_FE_TABLE + ' .' + VP_FE_TABLE_ROW)); $(document).off('click', this.wrapSelector('.' + VP_FE_TABLE + ' .' + VP_FE_TABLE_COLUMN)); @@ -485,38 +775,494 @@ define([ $(document).off('change', this.wrapSelector('.vp-inner-popup-var2')); } - bindEventForPopupPage() { + bindEventForPopupPage(menuType) { var that = this; - ///// add page - // 1. add type - $(this.wrapSelector('.vp-inner-popup-addtype')).on('change', function() { - var tab = $(this).val(); - $(that.wrapSelector('.vp-inner-popup-tab')).hide(); - $(that.wrapSelector('.vp-inner-popup-tab.' + tab)).show(); - }); - - // 2-1. hide column selection box - $(this.wrapSelector('.vp-inner-popup-var1box .vp-vs-data-type')).on('change', function() { - var type = $(this).val(); - if (type == 'DataFrame') { - $(that.wrapSelector('.vp-inner-popup-var1col')).show(); - } else { - $(that.wrapSelector('.vp-inner-popup-var1col')).hide(); + + if (menuType === FRAME_EDIT_TYPE.ADD_COL + || menuType === FRAME_EDIT_TYPE.ADD_ROW + || menuType === FRAME_EDIT_TYPE.REPLACE) { + // Add page + if (menuType === FRAME_EDIT_TYPE.ADD_COL + || menuType === FRAME_EDIT_TYPE.ADD_ROW) { + ///// add page + // 1. add type + $(this.wrapSelector('.vp-inner-popup-addtype')).on('change', function() { + var tab = $(this).val(); + $(that.wrapSelector('.vp-inner-popup-tab')).hide(); + $(that.wrapSelector('.vp-inner-popup-tab.' + tab)).show(); + + if (tab === 'apply') { + let colName = $(that.wrapSelector('.vp-inner-popup-apply-column option:selected')).text(); + $(that.wrapSelector('.vp-inner-popup-apply-target-name')).text(colName); + $(that.wrapSelector('.vp-inner-popup-apply-column')).trigger('change'); + } + }); + + // 2-1. hide column selection box + $(this.wrapSelector('.vp-inner-popup-var1box .vp-vs-data-type')).on('change', function() { + var type = $(this).val(); + if (type == 'DataFrame') { + $(that.wrapSelector('.vp-inner-popup-var1col')).show(); + } else { + $(that.wrapSelector('.vp-inner-popup-var1col')).hide(); + } + }); + + $(this.wrapSelector('.vp-inner-popup-var2box .vp-vs-data-type')).on('change', function() { + var type = $(this).val(); + if (type == 'DataFrame') { + $(that.wrapSelector('.vp-inner-popup-var2col')).show(); + } else { + $(that.wrapSelector('.vp-inner-popup-var2col')).hide(); + } + }); + + $(document).off('change', this.wrapSelector('.vp-inner-popup-vartype')); + $(document).on('change', this.wrapSelector('.vp-inner-popup-vartype'), function() { + var type = $(this).val(); + $(this).closest('tr').find('.vp-inner-popup-vartype-box').hide(); + $(this).closest('tr').find('.vp-inner-popup-vartype-box.' + type).show(); + }); } - }); + + // Replace page + if (menuType === FRAME_EDIT_TYPE.REPLACE) { + $(this.wrapSelector('.vp-inner-popup-replacetype')).on('change', function() { + var tab = $(this).val(); + $(that.wrapSelector('.vp-inner-popup-tab')).hide(); + $(that.wrapSelector('.vp-inner-popup-tab.' + tab)).show(); + + if (tab === 'apply') { + let colName = $(that.wrapSelector('.vp-inner-popup-apply-column option:selected')).text(); + $(that.wrapSelector('.vp-inner-popup-apply-target-name')).text(colName); + $(that.wrapSelector('.vp-inner-popup-apply-column')).trigger('change'); + } + }); + } + + // Add & Replace page + // condition add + $(document).off('click', this.wrapSelector('.vp-inner-popup-add-cond')); + $(document).on('click', this.wrapSelector('.vp-inner-popup-add-cond'), function (event) { + that.handleConditionAdd(); + }); + + // condition delete + $(document).off('click', this.wrapSelector('.vp-inner-popup-del-cond')); + $(document).on('click', this.wrapSelector('.vp-inner-popup-del-cond'), function (event) { + event.stopPropagation(); - $(this.wrapSelector('.vp-inner-popup-var2box .vp-vs-data-type')).on('change', function() { - var type = $(this).val(); - if (type == 'DataFrame') { - $(that.wrapSelector('.vp-inner-popup-var2col')).show(); - } else { - $(that.wrapSelector('.vp-inner-popup-var2col')).hide(); + // clear previous one + $(this).closest('tr').remove(); + $(that.wrapSelector('.vp-inner-popup-oper-connect:last')).hide(); + }); + + // change column selection for condition page + $(document).off('change', this.wrapSelector('.vp-inner-popup-col-list')); + $(document).on('change', this.wrapSelector('.vp-inner-popup-col-list'), function () { + var thisTag = $(this); + var varName = that.state.tempObj; + var colName = $(this).find('option:selected').attr('data-code'); + var colDtype = $(this).find('option:selected').attr('data-dtype'); + + var operTag = $(this).closest('td').find('.vp-inner-popup-oper-list'); + var condTag = $(this).closest('td').find('.vp-inner-popup-condition'); + + if (colName == '.index') { + // index + $(thisTag).closest('td').find('.vp-inner-popup-cond-use-text').prop('checked', false); + $(operTag).replaceWith(function () { + return that.templateForConditionOperator(''); + }); + $(condTag).replaceWith(function () { + return that.templateForConditionCondInput([], ''); + }); + that.generateCode(); + } else { + // get result and load column list + vpKernel.getColumnCategory(varName, colName).then(function (resultObj) { + let { result } = resultObj; + try { + var category = JSON.parse(result); + if (category && category.length > 0 && colDtype == 'object') { + // if it's categorical column and its dtype is object, check 'Text' as default + $(thisTag).closest('td').find('.vp-inner-popup-cond-use-text').prop('checked', true); + } else { + $(thisTag).closest('td').find('.vp-inner-popup-cond-use-text').prop('checked', false); + } + $(operTag).replaceWith(function () { + return that.templateForConditionOperator(colDtype); + }); + $(condTag).replaceWith(function () { + return that.templateForConditionCondInput(category, colDtype); + }); + } catch { + $(thisTag).closest('td').find('.vp-inner-popup-cond-use-text').prop('checked', false); + $(operTag).replaceWith(function () { + return that.templateForConditionOperator(colDtype); + }); + $(condTag).replaceWith(function () { + return that.templateForConditionCondInput([], colDtype); + }); + } + }); + } + }); + + // change operator selection + $(document).off('change', this.wrapSelector('.vp-inner-popup-oper-list')); + $(document).on('change', this.wrapSelector('.vp-inner-popup-oper-list'), function () { + var oper = $(this).val(); + var condTag = $(this).closest('td').find('.vp-inner-popup-condition'); + var useTextTag = $(this).closest('td').find('.vp-inner-popup-cond-use-text'); + // var colDtype = $(this).closest('td').find('.vp-col-list option:selected').attr('data-dtype'); + + // if operator is isnull(), notnull(), disable condition input + if (oper == 'isnull()' || oper == 'notnull()') { + $(condTag).prop('disabled', true); + $(useTextTag).prop('disabled', true); + } else { + $(condTag).prop('disabled', false); + $(useTextTag).prop('disabled', false); + } + }); + + // apply page + // 4. apply + $(this.wrapSelector('.vp-inner-popup-apply-column')).on('change', function() { + // TODO: change apply-condition (save value) + let colLabel = $(that.wrapSelector('.vp-inner-popup-apply-column option:selected')).text(); + $(that.wrapSelector('.vp-inner-popup-apply-target-name')).text(colLabel); + + var varName = that.state.tempObj; + let colName = $(this).val(); + var colDtype = $(this).find('option:selected').attr('data-dtype'); + var operTag = $(that.wrapSelector('.vp-inner-popup-apply-oper-list')); + var condTag = $(that.wrapSelector('.vp-inner-popup-apply-condition')); + // get result and load column list + vpKernel.getColumnCategory(varName, colName).then(function (resultObj) { + let { result } = resultObj; + try { + var category = JSON.parse(result); + if (category && category.length > 0 && colDtype == 'object') { + // if it's categorical column and its dtype is object, check 'Text' as default + $(that.wrapSelector('.vp-inner-popup-apply-cond-usetext')).prop('checked', true); + } else { + $(that.wrapSelector('.vp-inner-popup-apply-cond-usetext')).prop('checked', false); + } + $(operTag).replaceWith(function () { + return that.templateForConditionOperator(colDtype, 'vp-inner-popup-apply-oper-list', $(this).val()); + }); + $(condTag).replaceWith(function () { + return that.templateForConditionCondInput(category, colDtype, 'vp-inner-popup-apply-condition', $(this).val()); + }); + } catch { + $(that.wrapSelector('.vp-inner-popup-apply-cond-usetext')).prop('checked', false); + $(operTag).replaceWith(function () { + return that.templateForConditionOperator(colDtype, 'vp-inner-popup-apply-oper-list'); + }); + $(condTag).replaceWith(function () { + return that.templateForConditionCondInput([], colDtype, 'vp-inner-popup-apply-condition'); + }); + } + }); + }); + + // change operator selection + $(document).off('change', this.wrapSelector('.vp-inner-popup-apply-oper-list')); + $(document).on('change', this.wrapSelector('.vp-inner-popup-apply-oper-list'), function () { + var oper = $(this).val(); + var condTag = $(this).closest('td').find('.vp-inner-popup-apply-condition'); + var useTextTag = $(this).closest('td').find('.vp-inner-popup-apply-cond-usetext'); + + // if operator is isnull(), notnull(), disable condition input + if (oper == 'isnull()' || oper == 'notnull()') { + $(condTag).prop('disabled', true); + $(useTextTag).prop('disabled', true); + } else { + $(condTag).prop('disabled', false); + $(useTextTag).prop('disabled', false); + } + }); + + $(this.wrapSelector('.vp-inner-popup-toggle-else')).on('click', function() { + // toggle else on/off + let elseOn = $(this).attr('data-else'); // on / off + if (elseOn === 'off') { + // off -> on + $(this).attr('data-else', 'on'); + $(this).text('Else off'); + $(that.wrapSelector('.vp-inner-popup-apply-else-value')).prop('disabled', false); + $(that.wrapSelector('.vp-inner-popup-apply-else-usetext')).prop('disabled', false); + } else { + // on -> off + $(this).attr('data-else', 'off'); + $(this).text('Else on'); + $(that.wrapSelector('.vp-inner-popup-apply-else-value')).prop('disabled', true); + $(that.wrapSelector('.vp-inner-popup-apply-else-usetext')).prop('disabled', true); + } + }); + + $(this.wrapSelector('.vp-inner-popup-add-case')).on('click', function() { + // add case + $(this).parent().find('.vp-inner-popup-apply-case-box').append($(that.templateForApplyCase())); + $(that.wrapSelector('.vp-inner-popup-apply-column')).trigger('change'); + }); + + $(document).off('click', this.wrapSelector('.vp-inner-popup-apply-add-cond')); + $(document).on('click', this.wrapSelector('.vp-inner-popup-apply-add-cond'), function() { + // add condition + $(this).parent().find('.vp-inner-popup-apply-cond-box').append($(that.templateForApplyCondition())); + // show operator except last operator + $(this).parent().find('.vp-inner-popup-apply-oper-connect:not(:last)').show(); + $(that.wrapSelector('.vp-inner-popup-apply-column')).trigger('change'); + }); + + $(document).off('click', this.wrapSelector('.vp-inner-popup-apply-del-cond')); + $(document).on('click', this.wrapSelector('.vp-inner-popup-apply-del-cond'), function() { + let condBox = $(this).closest('.vp-inner-popup-apply-cond-box'); + // delete apply cond + $(this).parent().remove(); + // hide last operator + $(condBox).find('.vp-inner-popup-apply-oper-connect:last').hide(); + }); + + $(document).off('click', this.wrapSelector('.vp-inner-popup-apply-del-case')); + $(document).on('click', this.wrapSelector('.vp-inner-popup-apply-del-case'), function() { + // delete apply case + $(this).parent().remove(); + }); + } else if (menuType === FRAME_EDIT_TYPE.DISCRETIZE) { + // change bins + $(this.wrapSelector('.vp-inner-popup-bins')).on('change', function() { + let binsCount = $(this).val(); + that.handleDiscretizeEdges(binsCount); + }); + + // change cut to qcut(quantile based discretization) + $(this.wrapSelector('.vp-inner-popup-discretizetype')).on('change', function() { + let binsCount = $(that.wrapSelector('.vp-inner-popup-bins')).val(); + let discretizeType = $(this).val(); + // disable right and range table + if (discretizeType === 'qcut') { + $(that.wrapSelector('.vp-inner-popup-right')).prop('disabled', true); + $(that.wrapSelector('.vp-inner-popup-range-table input.vp-inner-popup-left-edge')).val(''); + $(that.wrapSelector('.vp-inner-popup-range-table input.vp-inner-popup-right-edge')).val(''); + $(that.wrapSelector('.vp-inner-popup-range-table input:not(.vp-inner-popup-label)')).prop('disabled', true); + } else { + $(that.wrapSelector('.vp-inner-popup-right')).prop('disabled', false); + $(that.wrapSelector('.vp-inner-popup-range-table input.vp-inner-popup-left-edge')).val(''); + $(that.wrapSelector('.vp-inner-popup-range-table input.vp-inner-popup-right-edge')).val(''); + $(that.wrapSelector('.vp-inner-popup-range-table input:not(.vp-inner-popup-label)')).prop('disabled', false); + } + that.handleDiscretizeEdges(binsCount); + }); + + // change right option + $(this.wrapSelector('.vp-inner-popup-right')).on('change', function() { + let binsCount = $(that.wrapSelector('.vp-inner-popup-bins')).val(); + let right = $(this).prop('checked'); + that.handleDiscretizeEdges(binsCount, right); + }); + } else if (menuType === FRAME_EDIT_TYPE.SORT_INDEX + || menuType === FRAME_EDIT_TYPE.SORT_VALUES) { + $(this.wrapSelector('.vp-inner-popup-axis')).on('change', function() { + let axis = $(this).val(); + if (axis === '0') { + axis = FRAME_AXIS.ROW; + } else { + axis = FRAME_AXIS.COLUMN; + } + $(that.wrapSelector('.vp-inner-popup-sortby')).replaceWith(that.templateForSortByBox('index', axis)); + // re-bind events + $(that.wrapSelector('.vp-inner-popup-sortby-up')).on('click', function() { + let tag = $(this).closest('.vp-inner-popup-sortby-item'); + tag.insertBefore(tag.prev()); + }); + $(that.wrapSelector('.vp-inner-popup-sortby-down')).on('click', function() { + let tag = $(this).closest('.vp-inner-popup-sortby-item'); + tag.insertAfter(tag.next()); + }); + }); + + $(this.wrapSelector('.vp-inner-popup-sortby-up')).on('click', function() { + let tag = $(this).closest('.vp-inner-popup-sortby-item'); + tag.insertBefore(tag.prev()); + }); + $(this.wrapSelector('.vp-inner-popup-sortby-down')).on('click', function() { + let tag = $(this).closest('.vp-inner-popup-sortby-item'); + tag.insertAfter(tag.next()); + }); + } else if (menuType === FRAME_EDIT_TYPE.FILL_OUT) { + $(this.wrapSelector('.vp-inner-popup-filltype')).on('change', function() { + let filltype = $(this).val(); + if (filltype === 'value') { + $(that.wrapSelector('.vp-inner-popup-fillvalue')).prop('disabled', false); + } else { + $(that.wrapSelector('.vp-inner-popup-fillvalue')).prop('disabled', true); + } + }); + } else if (menuType === FRAME_EDIT_TYPE.DROP_NA) { + $(this.wrapSelector('.vp-inner-popup-how')).on('change', function() { + let val = $(this).val(); + if (val === '') { + $(that.wrapSelector('.vp-inner-popup-thresh')).prop('disabled', false); + } else { + $(that.wrapSelector('.vp-inner-popup-thresh')).prop('disabled', true); + } + }); + } else if (menuType === FRAME_EDIT_TYPE.FILL_NA) { + // bind event on method + $(this.wrapSelector('.vp-inner-popup-method')).on('change', function() { + let changedVal = $(this).val(); + if (changedVal === 'value') { + // show value row + $(that.wrapSelector('.vp-inner-popup-value-row')).show(); + $(that.wrapSelector('.vp-inner-popup-fill-row')).hide(); + } else if (changedVal === 'ffill' || changedVal === 'bfill') { + // show method fill row + $(that.wrapSelector('.vp-inner-popup-value-row')).hide(); + $(that.wrapSelector('.vp-inner-popup-fill-row')).show(); + } else { + // hide all + $(that.wrapSelector('.vp-inner-popup-value-row')).hide(); + $(that.wrapSelector('.vp-inner-popup-fill-row')).hide(); + } + }); + } else if (menuType === FRAME_EDIT_TYPE.TO_DATETIME) { + // bind event for selecting format + $(this.wrapSelector('.vp-inner-popup-todt-format')).on('change', function() { + let format = $(this).val(); + if (format === 'auto') { + $(that.wrapSelector('.vp-inner-popup-todt-dayfirst')).prop('disabled', false); + } else { + $(that.wrapSelector('.vp-inner-popup-todt-dayfirst')).prop('disabled', true); + $(that.wrapSelector('.vp-inner-popup-todt-dayfirst')).val(''); + } + + if (format === 'typing') { + $(that.wrapSelector('.vp-inner-popup-todt-format-typing')).prop('disabled', false); + } else { + $(that.wrapSelector('.vp-inner-popup-todt-format-typing')).prop('disabled', true); + } + }); + + // Add column set event + $(this.wrapSelector('.vp-inner-popup-todt-addcol')).on('click', function() { + let dateTypeList = [ // df[col].dt[{dateType}] + { label: 'Year', value: 'year' }, + { label: 'Month', value: 'month' }, + { label: 'Day', value: 'day' }, + { label: 'Date', value: 'date' }, + { label: 'DayOfWeek', value: 'dayofweek' }, + { label: 'DayOfYear', value: 'dayofyear' }, + { label: 'DaysInMonth', value: 'daysinmonth' }, + { label: 'Quarter', value: 'quarter' }, + { label: 'Time', value: 'time' }, + { label: 'Hour', value: 'hour' }, + { label: 'Minute', value: 'minute' }, + { label: 'Second', value: 'second' }, + { label: 'Nanosecond', value: 'nanosecond' }, + ]; + let dateTypeOptionTag = new com_String(); + dateTypeList.forEach(opt => { + dateTypeOptionTag.appendFormat('', opt.value, opt.label); + }); + + let addColItemTag = $(``); + $(that.wrapSelector('.vp-inner-popup-todt-addcol-content')).append(addColItemTag); + $(addColItemTag)[0].scrollIntoView(); + + // bind event for selecting date type option + $(that.wrapSelector('.vp-inner-popup-todt-addcol-type')).off('change'); + $(that.wrapSelector('.vp-inner-popup-todt-addcol-type')).on('change', function() { + let dateTypeVal = $(this).val(); + $(this).parent().find('.vp-inner-popup-todt-addcol-colname').val(dateTypeVal); + }); + + // bind event for deleting + $(that.wrapSelector('.vp-inner-popup-todt-addcol-del')).off('click'); + $(that.wrapSelector('.vp-inner-popup-todt-addcol-del')).on('click', function() { + // delete item + $(this).closest('.vp-inner-popup-todt-addcol-item').remove(); + }); + }); + } + + } + + handleDiscretizeEdges(binsCount=1, right=true) { + let that = this; + $(this.wrapSelector('.vp-inner-popup-range-table tbody')).html(''); + $(this.wrapSelector('.vp-inner-popup-islabelchanged')).val("false"); + $(that.wrapSelector('.vp-inner-popup-isedgechanged')).val("false"); + + let code = new com_String(); + code.appendFormatLine("_out, _bins = pd.cut({0}[{1}], bins={2}, right={3}, retbins=True)" + , this.state.tempObj, this.state.selected[0].code, binsCount, right?'True':'False'); + code.append("_vp_print({'edges': list(_bins)})"); + vpKernel.execute(code.toString()).then(function(resultObj) { + let { result } = resultObj; + let { edges } = JSON.parse(result); + + let labelLength = edges.length - 1; + let edgeTbody = new com_String(); + for (let idx = 0; idx < labelLength; idx++ ) { + let leftDisabled = 'disabled'; + let rightDisabled = ''; + if (idx === (labelLength - 1)) { + rightDisabled = 'disabled'; + } + edgeTbody.append(''); + edgeTbody.appendFormatLine('', idx, idx); + edgeTbody.appendLine(':'); + edgeTbody.appendFormatLine('', idx, edges[idx], leftDisabled); + edgeTbody.appendLine('~'); + edgeTbody.appendFormatLine('', idx + 1, edges[idx+1], rightDisabled); + edgeTbody.append(''); } + $(that.wrapSelector('.vp-inner-popup-range-table tbody')).html(edgeTbody.toString()); + + // label change event + $(that.wrapSelector('.vp-inner-popup-label')).change(function() { + $(that.wrapSelector('.vp-inner-popup-islabelchanged')).val("true"); + }); + + // edge change event + $(that.wrapSelector('.vp-inner-popup-left-edge')).change(function() { + let idx = $(this).data('idx'); + let val = $(this).val(); + $(that.wrapSelector(`.vp-inner-popup-right-edge[data-idx=${idx}]`)).val(val); + $(that.wrapSelector('.vp-inner-popup-isedgechanged')).val("true"); + }); + $(that.wrapSelector('.vp-inner-popup-right-edge')).change(function() { + let idx = $(this).data('idx'); + let val = $(this).val(); + $(that.wrapSelector(`.vp-inner-popup-left-edge[data-idx=${idx}]`)).val(val); + $(that.wrapSelector('.vp-inner-popup-isedgechanged')).val("true"); + }); + + }).catch(function(errObj) { + vpLog.display(VP_LOG_TYPE.ERROR, errObj); }); } templateForBody() { - return frameHtml; + let page = $(frameHtml); + + // Removed dataselector for Allocation input + // let allocateSelector = new DataSelector({ + // pageThis: this, id: 'vp_feReturn', placeholder: 'Variable name', required: true, value: 'vp_df' + // }); + // $(page).find('#vp_feReturn').replaceWith(allocateSelector.toTagString()); + + return page; } render() { @@ -530,6 +1276,8 @@ define([ } = this.state; this.loadVariableList(); + + this.renderToolbar(); $(this.wrapSelector('#vp_feVariable')).val(originObj); @@ -543,6 +1291,9 @@ define([ // this.state.steps = []; this.loadCode(code); } + + // resize codeview + $(this.wrapSelector('.vp-popup-codeview-box')).css({'height': '300px'}) } renderVariableList(varList, defaultValue='') { @@ -565,14 +1316,76 @@ define([ }); } + /** + * Render toolbar and contextmenu + */ + renderToolbar() { + let that = this; + $(this.wrapSelector('.vp-fe-toolbox')).html(''); + $(this.wrapSelector('.vp-fe-menu-box')).html(''); + // add menu list + this.menuList & this.menuList.forEach((menuObj, idx) => { + // show menu list dynamically + let { id, label, child, axis, selection } = menuObj; + let enabled = true; + if ((that.state.axis !== FRAME_AXIS.NONE) && (axis !== undefined) && (axis !== FRAME_AXIS.NONE) && (that.state.axis !== axis)) { + enabled = false; + } + if (selection !== undefined && (selection !== FRAME_SELECT_TYPE.NONE)) { + if ((selection === FRAME_SELECT_TYPE.SINGLE) && (that.state.selected.length !== 1)) { + enabled = false; + } + if ((selection === FRAME_SELECT_TYPE.MULTI) && (that.state.selected.length === 0)) { + enabled = false; + } + } + let selected = id === that.state.menu; + let $toolbar = $(``); + let $menubox = ''; + if (child !== undefined) { + $menubox = $(``); + } else { + $menubox = $(``); + } + child && child.forEach(itemObj => { + let { id, label, menuType, axis, selection } = itemObj; + let enabled = true; + if ((that.state.axis !== FRAME_AXIS.NONE) && (axis !== undefined) && (axis !== FRAME_AXIS.NONE) && (that.state.axis !== axis)) { + enabled = false; + } + if (selection !== undefined && (selection !== FRAME_SELECT_TYPE.NONE)) { + if ((selection === FRAME_SELECT_TYPE.SINGLE) && (that.state.selected.length !== 1)) { + enabled = false; + } + if ((selection === FRAME_SELECT_TYPE.MULTI) && (that.state.selected.length === 0)) { + enabled = false; + } + } + let selected = that.state.menuItem === id; + $toolbar.find('.vp-dropdown-content') + .append($(``)); + $menubox.find('.vp-fe-menu-sub-box') + .append($(``)) + }); + $(this.wrapSelector('.vp-fe-toolbox')).append($toolbar); + $(this.wrapSelector('.vp-fe-menu-box')).append($menubox); + }); + } + renderTable(renderedText, isHtml=true) { var tag = new com_String(); // Table - tag.appendFormatLine('

', VP_FE_TABLE, 'rendered_html', 'vp-scrollbar'); + tag.appendFormatLine('

', VP_FE_TABLE, 'vp_rendered_html', 'vp-scrollbar'); if (isHtml) { tag.appendFormatLine('{0}

', renderedText); // More button - tag.appendFormatLine('

More...

', VP_FE_TABLE_MORE, 'vp-button'); + tag.appendFormatLine('

', VP_FE_TABLE_MORE, 'vp-button'); } else { tag.appendFormatLine('

{0}

', renderedText); } @@ -676,107 +1489,161 @@ define([ /** * Render Inner popup page - * @param {*} type - * @param {*} targetLabel + * @param {*} type * @returns */ - renderAddPage = function(type, targetLabel = '') { + renderAddPage(type = '') { var content = new com_String(); content.appendFormatLine('

', 'vp-inner-popup-addpage'); content.appendLine('

'); - content.appendLine(''); - content.appendFormatLine('', 'vp-orange-text', targetLabel); - var target = ''; - if (type == 'replace') { - target = this.state.selected.map(col => col.label).join(','); + content.appendLine('

{1}

'); + content.appendFormatLine('', 'vp-orange-text', type); + if (type === 'row') { + content.appendFormatLine(''); - content.appendLine(''); - content.appendFormatLine(''); + if (type === 'column' && this.state.columnLevel > 1) { + for (let i = 1; i < this.state.columnLevel; i++ ) { + content.appendLine(''); + content.appendFormatLine(''); + } } - if (type == 'column' || type == 'replace') { - content.appendFormatLine('', 'subset', 'Subset'); + if (type === 'column') { + content.appendLine(''); + content.appendFormatLine(''); } - content.appendFormatLine('', 'apply', 'Apply'); - content.appendLine(''); content.appendLine('

New {1}	', 'vp-inner-popup-input0', 'Type row name'); + } else { + content.appendFormatLine('	', 'vp-inner-popup-input0', 'level 0'); } - content.appendFormatLine('	', 'vp-inner-popup-input1', target, type=='replace'?'disabled':''); - content.appendFormatLine('{2}' - , 'vp-inner-popup-istext1', type=='replace'?'disabled':'', 'Text'); - content.appendLine('
Add Type	{1}' + , 'vp-inner-popup-inputastext0', 'Text'); + content.appendLine('
	', 'vp-inner-popup-input' + i, 'level ' + i); + content.appendFormatLine('{1}' + , 'vp-inner-popup-inputastext' + i, 'Text'); + content.appendLine('
Add type

'); content.appendLine('

'); // end of vp-inner-popup-header content.appendLine('

'); - // tab 1. value - content.appendFormatLine('

', 'vp-inner-popup-tab', 'value'); - content.appendLine(''); - content.appendLine(''); - content.appendFormatLine('

Value	', 'vp-inner-popup-input2'); - content.appendFormatLine('{1}', 'vp-inner-popup-istext2','Text'); - content.appendLine('

'); - content.appendLine('

'); // end of vp-inner-popup-tab value - - // tab 2. calculation - content.appendFormatLine('

', 'vp-inner-popup-tab', 'calculation'); - content.appendLine(''); - // calc - variable 1 - content.appendLine(''); - content.appendLine(''); - content.appendFormatLine('', 'vp-inner-popup-var1col'); - content.appendLine(''); - // calc -operator - content.appendLine(''); - content.appendLine(''); - content.appendFormatLine(''); - content.appendFormatLine('

Variable 1

Operator

'); + content.appendLine(''); + content.appendFormatLine('', 'vp-inner-popup-vartype'); + content.appendLine(''); - content.appendLine(''); - // calc - variable 2 - content.appendLine(''); - content.appendLine(''); - content.appendFormatLine('', 'vp-inner-popup-var2col'); - content.appendLine(''); + content.appendLine(''); + content.appendLine(''); + // content.appendFormatLine('', 'vp-inner-popup-delete-value'); + content.appendLine(''); + content.appendFormatLine('', 'vp-inner-popup-addvalue'); content.appendLine('

	'); + content.appendFormatLine(' ', 'vp-inner-popup-vartype-box variable'); + content.appendFormatLine('', 'vp-inner-popup-value', 0); + content.appendFormatLine('{1}', 'vp-inner-popup-istext','Text'); + content.appendLine(' '); + content.appendFormatLine(' ', 'vp-inner-popup-vartype-box column'); + content.appendFormatLine('
Variable 2

'); - content.appendLine(''); // end of vp-inner-popup-tab calculation - + content.appendLine(''); // end of vp-inner-popup-tab value + + // tab 2. statistics + content.appendFormatLine(''); + // tab 3. replace - content.appendFormatLine(''); // end of vp-inner-popup-tab replace - - // tab 4. subset - if (type == 'column' || type == 'replace') { - content.appendFormatLine(''); + content.appendLine(''); + + // tab 4. condition + // replace page - 2. condition + content.appendFormatLine(''); - content.appendLine('

Value

', 'vp-inner-popup-input3'); - content.appendFormatLine('{1}', 'vp-inner-popup-istext3','Text'); - content.appendLine('

'); - content.appendLine('

'); // end of vp-inner-popup-tab subset - } // tab 5. apply content.appendFormatLine(''); // end of vp-inner-popup-tab apply content.appendLine('

Column	{0}
Function

'); // end of vp-inner-popup-addpage @@ -785,28 +1652,160 @@ define([ return content.toString(); } + renderAddValueBox(idx) { + // add dataselector + let valueSelector = new DataSelector({ + pageThis: this, classes: 'vp-inner-popup-value', placeholder: 'Type value', withPopup: false + }); + $(this.wrapSelector('.vp-inner-popup-body')).find('.vp-inner-popup-value:nth(' + idx + ')').replaceWith(valueSelector.toTagString()); + } + + templateForApplyCase() { + return ``; + } + + templateForApplyCondition() { + let colLabel = $(this.wrapSelector('.vp-inner-popup-apply-column option:selected'))?.text(); + return ``; + } + + renderCalculator(idx) { + let content = new com_String(); + content.appendFormatLine(''); + return content.toString(); + } + renderColumnList = function(columnList) { var selectTag = new com_String(); selectTag.appendFormatLine(''); return selectTag.toString(); } - renderRenamePage = function() { + renderDropPage() { + var content = new com_String(); + content.appendFormatLine('

', 'vp-inner-popup-drop-page'); + content.appendFormatLine('Are you sure to delete {0} below?', (this.state.axis === FRAME_AXIS.COLUMN?'columns':'rows')); + content.appendFormatLine('

{0}

', this.state.selected.map(col=>col.code).join(', ')) + content.appendLine('

'); + // set content + $(this.wrapSelector('.vp-inner-popup-body')).html(content.toString()); + return content.toString(); + } + + /** + * Render rename page + * @param {string} type FRAME_AXIS + * @returns + */ + renderRenamePage = function(type = FRAME_AXIS.COLUMN) { var content = new com_String(); content.appendFormatLine('

', 'vp-inner-popup-rename-page', 'vp-scrollbar'); - content.appendLine(''); - content.appendLine(''); - this.state.selected.forEach((col, idx) => { - content.appendLine(''); - content.appendFormatLine('', col.label); - content.appendFormatLine(''); - }); + if (type === FRAME_AXIS.COLUMN && this.state.columnLevel > 1) { + content.appendFormatLine('

', 'vp-grid-col-110'); + content.appendLine('Level'); + content.appendFormatLine(''); + content.appendLine('

'); + content.appendLine('

'); + } + content.appendLine('

{0}	', 'vp-inner-popup-input' + idx); - content.appendFormatLine('{1}', 'vp-inner-popup-istext' + idx, 'Text'); - content.appendLine('

'); + content.appendLine(''); + content.appendLine(''); + if (this.state.columnLevel > 1) { + let selectedList = this.state.selected; + let selectedStr = ''; + if (selectedList.length === 0) { + // select all + selectedList = this.state.columnList; + selectedStr = selectedList.map(col => "(" + col.code.join(',') + ")").join(',') + } else { + selectedStr = selectedList.map(col => col.code).join(','); + } + let codeStr = com_util.formatString("_vp_print([ list(col) for col in {0}[[{1}]].columns.to_list()])" + , this.state.tempObj, selectedStr); + let that = this; + vpKernel.execute(codeStr).then(function(resultObj) { + let { result } = resultObj; + let colList = JSON.parse(result); + let colTags = new com_String(); + for (let i = 0; i < colList.length; i++) { + let colLevels = colList[i]; + for (let j = 0; j < colLevels.length; j++) { + let idx = (i + 1) * j + colTags.appendFormatLine('', 'vp-inner-popup-input-row'); + colTags.appendFormatLine('', j * 10, colLevels[j]); + colTags.appendFormatLine('', 'vp-inner-popup-istext' + idx, 'Text'); + colTags.appendLine(''); + } + } + $(that.wrapSelector('.vp-inner-popup-rename-page tbody')).html(colTags.toString()); + }); + } else { + let selectedList = this.state.selected; + if (selectedList.length === 0) { + // select all + selectedList = this.state.columnList; + } + selectedList.forEach((col, idx) => { + content.appendFormatLine('', 'vp-inner-popup-input-row'); + content.appendFormatLine('', col.label); + content.appendFormatLine('' + , 'vp-inner-popup-istext' + idx, 'Text'); + content.appendLine(''); + }); + } + content.appendLine(''); content.appendLine('

{1}	' + , 'vp-inner-popup-input' + idx, com_util.convertToStr(colLevels[j], true)); // FIXME: text or num ? + colTags.appendFormatLine('{1}
{0}	' + , 'vp-inner-popup-input' + idx, col.code); + content.appendFormatLine('{1}

'); content.appendLine('

'); @@ -815,16 +1814,251 @@ define([ return content.toString(); } + renderDiscretizePage() { + var content = new com_String(); + content.appendLine(` +

+ New column +

+ + + + Text + +

+ Target column + + Bins count + + Discretize type + +

+ + + Include the rightmost edge + +

+ + + + + + + + + + + + + + +

+ + + Label as Text + +

+ + +

+ `) + + // set content + $(this.wrapSelector('.vp-inner-popup-body')).html(content.toString()); + return content.toString(); + } + + renderShiftPage() { + var content = new com_String(); + content.appendFormatLine('

', 'vp-inner-popup-shift-page'); + content.appendLine(''); + content.appendLine(''); + content.appendLine(''); + content.appendFormatLine('', 'Periods'); + content.appendFormatLine('' + , 'vp-inner-popup-periods', 'Type number'); + content.appendLine(''); + content.appendFormatLine('' + , 'NOTE:', 'Number of periods to shift. Can be positive or negative.'); + content.appendLine(''); + content.appendLine(''); + content.appendFormatLine('', 'Frequency'); + content.appendFormatLine('' + , 'vp-inner-popup-freq', 'Offset for timeseries'); + content.appendLine(''); + content.appendLine(''); + content.appendFormatLine('', 'Fill value'); + content.appendLine(''); + content.appendLine(''); + content.appendLine('

{0}
{0} {1}
{0}
{0}	'); + content.appendFormatLine('' + , 'vp-inner-popup-fillvalue', 'Type value to fill'); + content.appendFormatLine('{1}', 'vp-inner-popup-fillvalueastext', 'Text'); + content.appendLine('

'); + content.appendLine('

'); + + // set content + $(this.wrapSelector('.vp-inner-popup-body')).html(content.toString()); + return content.toString(); + } + + /** + * + * @param {int} method index / values + * @returns + */ + renderSortPage(method='values') { + var content = new com_String(); + content.appendFormatLine('

', 'vp-inner-popup-sort-page'); + content.appendLine('

'); + // axis + let sortByStr = 'column'; + if (method === 'index') { + content.appendFormatLine('{0}', 'Axis'); + content.appendFormatLine(''); + sortByStr = 'level'; + } + // sort by + content.appendFormatLine('{0} {1}', 'Sort by', sortByStr); + content.appendLine(this.templateForSortByBox(method)); + + // ascending + content.appendFormatLine('{0}', 'Ascending'); + content.appendFormatLine(''); + content.appendLine('

'); + content.appendLine('

'); + // set content + $(this.wrapSelector('.vp-inner-popup-body')).html(content.toString()); + return content.toString(); + } + + templateForSortByBox(method='index', axis=FRAME_AXIS.ROW) { + var content = new com_String(); + let sortByList = []; + if (method === 'values') { + // sort_values + sortByList = this.state.selected; + } else { + // sort_index + if (axis === FRAME_AXIS.ROW) { + sortByList = Array.from({ length:this.state.indexLevel },(v,k)=>{ return {label: k, code: k} }); + } else { + sortByList = Array.from({ length:this.state.columnLevel },(v,k)=>{ return {label: k, code: k} }); + } + } + + // movable list + content.appendLine(''); + return content.toString(); + } + renderReplacePage() { var content = new com_String(); + content.appendFormatLine('

', 'vp-inner-popup-replacepage'); + content.appendLine('

'); + content.appendLine(''); + content.appendFormatLine('', '', 'Column'); + var target = this.state.selected[0]; + content.appendFormatLine(''); + content.appendLine(''); + content.appendFormatLine(''); + content.appendLine('

{1}	' + , 'vp-inner-popup-input1', target.label, target.code); + content.appendLine('
Replace type

'); + content.appendLine('

'); // end of vp-inner-popup-header + + content.appendLine('

'); + // replace page - 1. replace + content.appendFormatLine('

', 'vp-inner-popup-tab replace'); content.appendFormatLine('{1}', 'vp-inner-popup-use-regex', 'Use Regular Expression'); content.appendLine('

'); content.appendFormatLine('

', 'vp-inner-popup-replace-table'); - content.appendLine(''); + content.appendLine('

'); + content.appendLine(''); + content.appendFormatLine('', 'Value', 'New value'); + content.appendLine(''); content.appendLine(this.renderReplaceInput(0)); - content.appendFormatLine('', 'vp-button', 'vp-inner-popup-replace-add', '+ Add Key'); + content.appendFormatLine('', 'vp-button', 'vp-inner-popup-replace-add', '+ Add value'); + content.appendLine(''); content.appendLine('

{0}	{1}

'); content.appendLine('

'); + content.appendLine('

'); + // replace page - 2. condition + content.appendFormatLine(''); // end of vp-inner-popup-tab condition + + // replace page - 3. apply + content.appendFormatLine(''); // end of vp-inner-popup-tab apply + + content.appendLine('

'); // end of vp-inner-popup-addpage + + // set content + $(this.wrapSelector('.vp-inner-popup-body')).html(content.toString()); return content.toString(); } @@ -846,16 +2080,107 @@ define([ return content.toString(); } + handleConditionAdd() { + var conditonBox = $(this.renderReplaceCondition()); + + // hide last connect operator + conditonBox.find('.vp-inner-popup-oper-connect').hide(); + + // show connect operator right before last one + $(this.wrapSelector('.vp-inner-popup-oper-connect:last')).show(); + conditonBox.insertBefore(this.wrapSelector('.vp-inner-popup-condition-tbl tr:last')); + } + + renderReplaceCondition() { + var content = new com_String(); + content.appendLine(''); + // delete condition button + content.appendLine(''); + // condition line + content.appendLine('

'); + // - 1. column list + content.appendFormatLine(''); + // - 2. operator + content.appendLine(this.templateForConditionOperator('')); + // - 3. oper-value + content.appendLine(''); + // use text + content.appendFormatLine('{3}', + 'vp-inner-popup-condition-use-text', 'vp-inner-popup-cond-use-text', 'Uncheck it if you want to use variable or numeric values.', 'Text'); + content.appendLine('

'); + content.appendLine('

'); + content.appendLine(''); + content.appendLine('

'); + content.appendLine(''); + return content.toString(); + } + + templateForConditionOperator(dtype='object', className='vp-inner-popup-oper-list', prevValue='') { + var content = new com_String(); + content.appendFormatLine(''); + return content.toString(); + } + + templateForConditionCondInput(category, dtype='object', className='vp-inner-popup-condition', prevValue='') { + var vpCondSuggest = new SuggestInput(); + vpCondSuggest.addClass('vp-input m ' + className); + if (prevValue !== '') { + vpCondSuggest.setValue(prevValue); + } + if (category && category.length > 0) { + vpCondSuggest.setPlaceholder((dtype=='object'?'Categorical':dtype) + " dtype"); + vpCondSuggest.setSuggestList(function () { return category; }); + vpCondSuggest.setSelectEvent(function (value) { + $(this.wrapSelector()).val(value); + $(this.wrapSelector()).trigger('change'); + }); + vpCondSuggest.setNormalFilter(false); + } else { + vpCondSuggest.setPlaceholder(dtype==''?'Value':(dtype + " dtype")); + } + return vpCondSuggest.toTagString(); + } + renderAsType() { var astypeList = this.astypeList; var content = new com_String(); content.appendFormatLine('

', 'vp-inner-popup-astype'); - content.appendFormatLine('', 'vp-inner-popup-astype-table'); - content.appendLine(''); + content.appendFormatLine('

', 'vp-inner-popup-astype-table'); + content.appendLine(''); content.appendFormatLine('' , 'Column', 'Data type', 'vp-orange-text', 'New data type'); content.appendLine(''); - this.state.selected.forEach((col, idx) => { + let selectedList = this.state.selected; + if (selectedList.length === 0) { + // select all + selectedList = this.state.columnList; + } + selectedList.forEach((col, idx) => { content.appendLine(''); content.appendFormatLine('', col.label, col.label); content.appendFormatLine('', col.type); @@ -875,111 +2200,456 @@ define([ return content.toString(); } - openInputPopup(type, width=400, height=400) { + renderToDatetime() { + var content = new com_String(); + let formatList = [ + { label: 'Auto', value: 'auto' }, + { label: '%Y/%m/%d', value: '%Y/%m/%d' }, + { label: '%Y-%m-%d', value: '%Y-%m-%d' }, + { label: '%y/%m/%d', value: '%y/%m/%d' }, + { label: '%y-%m-%d', value: '%y-%m-%d' }, + { label: '%d/%m/%Y', value: '%d/%m/%Y' }, + { label: '%d-%m-%Y', value: '%d-%m-%Y' }, + { label: '%d/%m/%y', value: '%d/%m/%y' }, + { label: '%d-%m-%y', value: '%d-%m-%y' }, + { label: 'Typing', value: 'typing' } + ]; + let formatOptionTag = new com_String(); + formatList.forEach(opt => { + formatOptionTag.appendFormat('', opt.value, opt.label); + }); + + content.appendFormat(` + `, this.state.selected[0].label, formatOptionTag.toString(), this.state.selected[0].label); + + // set content + $(this.wrapSelector('.vp-inner-popup-body')).html(content.toString()); + return content.toString(); + } + + renderFillNAPage() { + var content = new com_String(); + content.appendFormatLine('

', 'vp-inner-popup-fillna-page'); + content.appendLine('

{0}	{1}	{3}
{1}

'); + content.appendLine(''); + content.appendLine(''); + content.appendFormatLine('', 'Method'); + content.appendFormatLine(''); + content.appendLine(''); + content.appendFormatLine('', 'vp-inner-popup-value-row'); + content.appendFormatLine('', 'Fill value'); + content.appendLine(''); + content.appendLine(''); + content.appendFormatLine('', 'vp-inner-popup-fill-row'); + content.appendFormatLine('', 'Limit'); + content.appendLine(''); + content.appendLine(''); + content.appendLine('

{0}
{0}	'); + content.appendFormatLine('' + , 'vp_fillValue', 'vp-inner-popup-value', 'Type or select value'); + content.appendFormatLine('{1}' + , 'vp-inner-popup-valueastext', 'Text'); + content.appendLine('
{0}	'); + content.appendFormatLine('' + , 'vp-inner-popup-limit', 'Type limit to fill'); + content.appendLine('

'); + content.appendLine('

'); + + // set content + $(this.wrapSelector('.vp-inner-popup-body')).html(content.toString()); + return content.toString(); + } + + renderDropNAPage() { + // how / thresh / ignore_index + let content = ``; + + // set content + $(this.wrapSelector('.vp-inner-popup-body')).html(content); + return content.toString(); + } + + renderDropDupPage() { + // keep / ignore_index + let content = ``; + + // set content + $(this.wrapSelector('.vp-inner-popup-body')).html(content); + return content.toString(); + } + + renderFillOutPage() { + let content = ``; + + // set content + $(this.wrapSelector('.vp-inner-popup-body')).html(content); + return content.toString(); + } + + + renderDropOutPage() { + var content = new com_String(); + content.appendFormatLine('

', 'vp-inner-popup-dropout-page'); + content.appendLine('Are you sure to drop outliers from columns below?'); + content.appendFormatLine('

{0}

', this.state.selected.map(col=>col.code).join(', ')) + content.appendLine('

'); + + // set content + $(this.wrapSelector('.vp-inner-popup-body')).html(content.toString()); + return content.toString(); + } + + openInputPopup(type, width=450, height=450) { var title = ''; var content = ''; let size = { width: width, height: height }; let that = this; + + $(this.wrapSelector('.vp-inner-popup-body')).empty(); switch (parseInt(type)) { case FRAME_EDIT_TYPE.ADD_COL: - title = 'Add Column'; - content = this.renderAddPage('column', 'Column Name'); - - // bind codemirror - this.subsetCm = this.initCodemirror({ - key: 'vp-inner-popup-subset', - selector: this.wrapSelector('.vp-inner-popup-subset'), - type: 'readonly' + title = 'Add column'; + size = { width: 550, height: 480 }; + content = this.renderAddPage('column'); + this.renderAddValueBox(0); + + // bind event for adding values to calculate + $(this.wrapSelector('.vp-inner-popup-addvalue')).on('click', function() { + let valueCount = $(that.wrapSelector('.vp-inner-popup-value')).length; + let columnOptStr = ''; + that.state.columnList.forEach(col => { + columnOptStr += ``; + }); + $(` + + ${that.renderCalculator(valueCount)} + + + + + + + + + + + + + `).insertBefore($(that.wrapSelector('.vp-inner-popup-addvalue-row'))); + that.renderAddValueBox(valueCount); + + $(that.wrapSelector('.vp-inner-popup-delete-value')).off('click'); + $(that.wrapSelector('.vp-inner-popup-delete-value')).on('click', function() { + // delete variable item + let index = $(this).closest('tr.vp-inner-popup-value-row').index(); + $(that.wrapSelector('.vp-inner-popup-oper-row:nth(' + (index - 2) + ')')).remove(); + $(that.wrapSelector('.vp-inner-popup-value-row:nth(' + (index - 1) + ')')).remove(); + }); }); - // set subset - let contentState = that.getPopupContent(type); - this.subsetEditor = new Subset({ - pandasObject: this.state.tempObj, - selectedColumns: [ com_util.convertToStr(contentState.name, contentState.nameastext) ], - config: { name: 'Subset' } }, - { - useInputVariable: true, - useInputColumns: true, - targetSelector: this.wrapSelector('.vp-inner-popup-subset'), - pageThis: this, - allowSubsetTypes: ['iloc', 'loc'], - beforeOpen: function(subsetThis) { - let contentState = that.getPopupContent(type); - let name = com_util.convertToStr(contentState.name, contentState.nameastext); - subsetThis.state.selectedColumns = [ name ]; - }, - finish: function(code) { - that.subsetCm.setValue(code); - that.subsetCm.save(); - setTimeout(function () { - that.subsetCm.refresh(); - }, 1); - } + + // bind multiselector + this.statsSelector = new MultiSelector(this.wrapSelector('.vp-inner-popup-stats-col-list'), + { mode: 'columns', parent: this.state.tempObj, showDescription: false } + ); + + // bind codemirror for apply textarea + this.applyCm = this.initCodemirror({ + key: 'vp-inner-popup-apply-lambda', + selector: this.wrapSelector('.vp-inner-popup-apply-lambda'), }); - // initial code - var code = this.subsetEditor.generateCode(); - that.subsetCm.setValue(code); - that.subsetCm.save(); - setTimeout(function () { - that.subsetCm.refresh(); - }, 1); - break; case FRAME_EDIT_TYPE.ADD_ROW: - title = 'Add Row'; - content = this.renderAddPage('row', 'Row Name'); + title = 'Add row'; + size = { width: 450, height: 450 }; + content = this.renderAddPage('row'); + this.renderAddValueBox(0); + + // bind event for adding values to calculate + $(this.wrapSelector('.vp-inner-popup-addvalue')).on('click', function() { + let valueCount = $(that.wrapSelector('.vp-inner-popup-value')).length; + $(` + + ${that.renderCalculator(valueCount)} + + + Variable + + + Text + + + `).insertBefore($(that.wrapSelector('.vp-inner-popup-addvalue-row'))); + that.renderAddValueBox(valueCount); + + $(that.wrapSelector('.vp-inner-popup-delete-value')).off('click'); + $(that.wrapSelector('.vp-inner-popup-delete-value')).on('click', function() { + // delete variable item + let index = $(this).closest('tr.vp-inner-popup-value-row').index(); + $(that.wrapSelector('.vp-inner-popup-oper-row:nth(' + (index - 2) + ')')).remove(); + $(that.wrapSelector('.vp-inner-popup-value-row:nth(' + (index - 1) + ')')).remove(); + }); + }); + break; + case FRAME_EDIT_TYPE.DROP: + title = 'Drop '; + if (this.state.axis === FRAME_AXIS.COLUMN) { + title += 'columns'; + } else { + title += 'rows'; + } + size = { width: 400, height: 200 }; + content = this.renderDropPage(); break; case FRAME_EDIT_TYPE.RENAME: - title = 'Rename'; - content = this.renderRenamePage(); + title = 'Rename '; + if (this.state.axis === FRAME_AXIS.ROW) { + title += 'rows'; + content = this.renderRenamePage(FRAME_AXIS.ROW); + } else { + title += 'columns'; + content = this.renderRenamePage(FRAME_AXIS.COLUMN); + } + break; + case FRAME_EDIT_TYPE.DISCRETIZE: + title = 'Discretize'; + size = { width: 450, height: 450 }; + content = this.renderDiscretizePage(); + break; + case FRAME_EDIT_TYPE.DATA_SHIFT: + title = 'Data shift'; + size = { width: 450, height: 300 }; + content = this.renderShiftPage(); + + // set suggestinput + let freqFormats = [ + {'label': 'infer', 'value': 'infer'}, + {'label': 'second', 'value': 's'}, + {'label': 'minute', 'value': 'T'}, + {'label': 'hour', 'value': 'H'}, + {'label': 'day', 'value': 'D'}, + {'label': 'weekdays', 'value': 'B'}, + {'label': 'week(Sunday)', 'value': 'W'}, + {'label': 'week(Monday)', 'value': 'W-MON'}, + {'label': 'first day of month', 'value': 'MS'}, + {'label': 'last day of month', 'value': 'M'}, + {'label': 'first weekday of month', 'value': 'BMS'}, + {'label': 'last weekday of month', 'value': 'BM'} + ]; + var freqInput = new SuggestInput(); + freqInput.addClass('vp-inner-popup-freq'); + freqInput.setPlaceholder('Type frequency'); + freqInput.setSuggestList(freqFormats); + freqInput.setNormalFilter(true); + $(this.wrapSelector('.vp-inner-popup-freq')).replaceWith(function() { + return freqInput.toTagString(); + }); + break; + case FRAME_EDIT_TYPE.SORT_INDEX: + title = 'Sort by index'; + content = this.renderSortPage('index'); + break; + case FRAME_EDIT_TYPE.SORT_VALUES: + title = 'Sort by values'; + content = this.renderSortPage('values'); break; case FRAME_EDIT_TYPE.REPLACE: title = 'Replace'; - // content = this.renderReplacePage(); - content = this.renderAddPage('replace', 'Column'); - - // bind codemirror - this.subsetCm = this.initCodemirror({ - key: 'vp-inner-popup-subset', - selector: this.wrapSelector('.vp-inner-popup-subset'), - type: 'readonly' - }); - // set subset - this.subsetEditor = new Subset({ - pandasObject: this.state.tempObj, - selectedColumns: that.state.selected.map(col=>col.code), - config: { name: 'Subset' } }, - { - useInputVariable: true, - useInputColumns: true, - targetSelector: this.wrapSelector('.vp-inner-popup-subset'), - pageThis: this, - allowSubsetTypes: ['iloc', 'loc'], - beforeOpen: function(subsetThis) { - subsetThis.state.selectedColumns = that.state.selected.map(col=>col.code); - }, - finish: function(code) { - that.subsetCm.setValue(code); - that.subsetCm.save(); - setTimeout(function () { - that.subsetCm.refresh(); - }, 1); - } - }); + content = this.renderReplacePage(); + size = { width: 550, height: 480 }; + + // // bind codemirror + // this.subsetCm = this.initCodemirror({ + // key: 'vp-inner-popup-subset', + // selector: this.wrapSelector('.vp-inner-popup-subset'), + // type: 'readonly' + // }); + // // set subset + // let contentState = that.getPopupContent(type); + // this.subsetEditor = new Subset({ + // pandasObject: this.state.tempObj, + // selectedColumns: [ com_util.convertToStr(contentState.name, contentState.nameastext) ], + // config: { name: 'Subset' } }, + // { + // useInputVariable: true, + // useInputColumns: true, + // targetSelector: this.wrapSelector('.vp-inner-popup-subset'), + // pageThis: this, + // allowSubsetTypes: ['iloc', 'loc'], + // beforeOpen: function(subsetThis) { + // let contentState = that.getPopupContent(type); + // let name = com_util.convertToStr(contentState.name, contentState.nameastext); + // subsetThis.state.selectedColumns = [ name ]; + // }, + // finish: function(code) { + // that.subsetCm.setValue(code); + // that.subsetCm.save(); + // setTimeout(function () { + // that.subsetCm.refresh(); + // }, 1); + // } + // }); // initial code - var code = this.subsetEditor.generateCode(); - that.subsetCm.setValue(code); - that.subsetCm.save(); - setTimeout(function () { - that.subsetCm.refresh(); - }, 1); - + // var code = this.subsetEditor.generateCode(); + // this.subsetCm.setValue(code); + // this.subsetCm.save(); + // setTimeout(function () { + // that.subsetCm.refresh(); + // }, 1); + + // set dataselector + let replaceVarSelector = new DataSelector({ + pageThis: this, id: 'vp_replaceVariable', classes: 'vp-inner-popup-value', placeholder: 'Type or select variable' + , withPopup: false + }); + $(this.wrapSelector('.vp-inner-popup-body')).find('.vp-inner-popup-value').replaceWith(replaceVarSelector.toTagString()); break; case FRAME_EDIT_TYPE.AS_TYPE: title = 'Convert type'; content = this.renderAsType(); break; + case FRAME_EDIT_TYPE.TO_DATETIME: + title = 'Convert to Datetime'; + size = { width: 500, height: 450 }; + content = this.renderToDatetime(); + break; + case FRAME_EDIT_TYPE.FILL_NA: + title = 'Fill NA'; + content = this.renderFillNAPage(); + + // set dataselector + let valueSelector = new DataSelector({ + pageThis: this, classes: 'vp-inner-popup-value', placeholder: 'Type or select value', + withPopup: false + }); + $(this.wrapSelector('.vp-inner-popup-body')).find('.vp-inner-popup-value').replaceWith(valueSelector.toTagString()); + break; + case FRAME_EDIT_TYPE.DROP_NA: + title = 'Drop NA'; + content = this.renderDropNAPage(); + break; + case FRAME_EDIT_TYPE.DROP_DUP: + title = 'Drop duplicates'; + content = this.renderDropDupPage(); + break; + case FRAME_EDIT_TYPE.FILL_OUT: + title = 'Fill outlier'; + content = this.renderFillOutPage(); + + // set dataselector + let fillvalueSelector = new DataSelector({ + pageThis: this, classes: 'vp-inner-popup-fillvalue', placeholder: 'Type or select value', + withPopup: false + }); + $(this.wrapSelector('.vp-inner-popup-body')).find('.vp-inner-popup-fillvalue').replaceWith(fillvalueSelector.toTagString()); + $(this.wrapSelector('.vp-inner-popup-body')).find('.vp-inner-popup-fillvalue').prop('disabled', true); + break; + case FRAME_EDIT_TYPE.DROP_OUT: + title = 'Drop outlier'; + size = { width: 400, height: 200 }; + content = this.renderDropOutPage(); + break; default: type = FRAME_EDIT_TYPE.NONE; break; @@ -987,21 +2657,26 @@ define([ this.state.popup.type = type; - // set size - $(this.wrapSelector('.vp-inner-popup-box')).css(size); + // set size and position + $(this.wrapSelector('.vp-inner-popup-box')).css({ + width: size.width, + height: size.height, + left: 'calc(50% - ' + (size.width/2) + 'px)', + top: 'calc(50% - ' + (size.height/2) + 'px)', + }); // bindEventForAddPage - this.bindEventForPopupPage(); + this.bindEventForPopupPage(type); // set column list vpKernel.getColumnList(this.state.tempObj).then(function(resultObj) { let { result } = resultObj; - var colList = JSON.parse(result); + var { list } = JSON.parse(result); var tag1 = new com_String(); var tag2 = new com_String(); tag1.appendFormatLine('', 'vp-inner-popup-var2col'); - colList && colList.forEach(col => { + list && list.forEach(col => { tag1.appendFormatLine('' , col.value, col.label, col.label); tag2.appendFormatLine('' @@ -1023,26 +2698,57 @@ define([ } getPopupContent = function(type) { + let that = this; var content = {}; switch (type) { case FRAME_EDIT_TYPE.ADD_COL: - case FRAME_EDIT_TYPE.REPLACE: - content['name'] = $(this.wrapSelector('.vp-inner-popup-input1')).val(); - if (content['name'] == '') { - $(this.wrapSelector('.vp-inner-popup-input1')).attr({'placeholder': 'Required input'}); - $(this.wrapSelector('.vp-inner-popup-input1')).focus(); + case FRAME_EDIT_TYPE.ADD_ROW: + let variableTuple = []; + let thisLevel = this.state.columnLevel; + if (type === FRAME_EDIT_TYPE.ADD_ROW) { + thisLevel = this.state.indexLevel; + } + for (let i = 0; i < thisLevel; i++) { + let val = $(this.wrapSelector('.vp-inner-popup-input' + i)).val(); + let istext = $(this.wrapSelector('.vp-inner-popup-inputastext' + i)).prop('checked'); + variableTuple.push(com_util.convertToStr(val, istext)); + } + if (variableTuple.length > 1) { + content['name'] = '(' + variableTuple.join(',') + ')'; + } else { + content['name'] = variableTuple.join(','); } var tab = $(this.wrapSelector('.vp-inner-popup-addtype')).val(); - content['nameastext'] = $(this.wrapSelector('.vp-inner-popup-istext1')).prop('checked'); + if (type === FRAME_EDIT_TYPE.ADD_ROW) { + tab = 'calculate'; + } content['addtype'] = tab; - if (tab == 'value') { - content['value'] = $(this.wrapSelector('.vp-inner-popup-input2')).val(); - content['valueastext'] = $(this.wrapSelector('.vp-inner-popup-istext2')).prop('checked'); - } else if (tab == 'calculation') { - content['var1col'] = $(this.wrapSelector('.vp-inner-popup-var1col')).val(); - content['oper'] = $(this.wrapSelector('.vp-inner-popup-oper')).val(); - content['var2col'] = $(this.wrapSelector('.vp-inner-popup-var2col')).val(); + if (tab == 'calculate') { + let values = []; + let opers = []; + $(this.wrapSelector('.vp-inner-popup-tab.calculate tr.vp-inner-popup-value-row')).each((idx, tag) => { + let varType = $(tag).find('.vp-inner-popup-vartype').val(); + if (varType === 'variable') { + let valueastext = $(tag).find('.vp-inner-popup-istext').prop('checked'); + values.push(com_util.convertToStr($(tag).find('.vp-inner-popup-value').val(), valueastext)); + } else { + // columns + let column = $(tag).find('.vp-inner-popup-var-col-list').val(); + values.push(com_util.formatString("{0}[{1}]", this.state.tempObj, column)); + } + let oper = $(that.wrapSelector('.vp-inner-popup-oper:nth(' + idx + ')')).val(); + if (oper && oper !== '') { + opers.push(oper); + } + }); + content['values'] = values; + content['opers'] = opers; + } else if (tab == 'statistics') { + var colList = this.statsSelector.getDataList(); + content['columns'] = colList.map(x => x.code); + content['method'] = $(this.wrapSelector('.vp-inner-popup-stats-method')).val(); } else if (tab == 'replace') { + content['target'] = $(this.wrapSelector('.vp-inner-popup-value-col-list option:selected')).data('code'); var useregex = $(this.wrapSelector('.vp-inner-popup-use-regex')).prop('checked'); content['useregex'] = useregex; content['list'] = []; @@ -1060,60 +2766,294 @@ define([ }); } } - } else if (tab == 'subset') { - content['subset'] = this.subsetCm?this.subsetCm.getValue():''; - content['value'] = $(this.wrapSelector('.vp-inner-popup-input3')).val(); - content['valueastext'] = $(this.wrapSelector('.vp-inner-popup-istext3')).prop('checked'); } else if (tab == 'apply') { - content['column'] = $(this.wrapSelector('.vp-inner-popup-apply-column')).val(); - content['apply'] = $(this.wrapSelector('.vp-inner-popup-apply-lambda')).val(); + content['target'] = $(this.wrapSelector('.vp-inner-popup-apply-column')).val(); + let caseList = []; + $(this.wrapSelector('.vp-inner-popup-apply-case-item')).each((idx, caseTag) => { + let condList = []; + let replaceValue = $(caseTag).find('.vp-inner-popup-apply-case-val').val(); + let replaceValText = $(caseTag).find('.vp-inner-popup-apply-case-usetext').prop('checked'); + + let operTag = $(caseTag).find('.vp-inner-popup-apply-oper-list'); + let condTag = $(caseTag).find('.vp-inner-popup-apply-condition'); + let condTextTag = $(caseTag).find('.vp-inner-popup-apply-cond-usetext'); + let operConnTag = $(caseTag).find('.vp-inner-popup-apply-oper-connect'); + for (let i=0; i { + let condList = []; + let replaceValue = $(caseTag).find('.vp-inner-popup-apply-case-val').val(); + let replaceValText = $(caseTag).find('.vp-inner-popup-apply-case-usetext').prop('checked'); + + let operTag = $(caseTag).find('.vp-inner-popup-apply-oper-list'); + let condTag = $(caseTag).find('.vp-inner-popup-apply-condition'); + let condTextTag = $(caseTag).find('.vp-inner-popup-apply-cond-usetext'); + let operConnTag = $(caseTag).find('.vp-inner-popup-apply-oper-connect'); + for (let i=0; i { + content['list'] = {}; + let inputLength = $(this.wrapSelector('.vp-inner-popup-input-row')).length; + for (let idx = 0; idx < inputLength; idx++) { + var label = $(this.wrapSelector('.vp-inner-popup-input'+idx)).data('code'); var value = $(this.wrapSelector('.vp-inner-popup-input'+idx)).val(); var istext = $(this.wrapSelector('.vp-inner-popup-istext'+idx)).prop('checked'); - content[idx] = { - label: element.code, + content['list'][idx] = { + label: label, value: value, istext: istext }; + } + if (this.state.axis !== FRAME_AXIS.ROW && this.state.columnLevel > 1) { + content['level'] = $(this.wrapSelector('.vp-inner-popup-level')).val(); + } + break; + case FRAME_EDIT_TYPE.SORT_INDEX: + content['axis'] = $(this.wrapSelector('.vp-inner-popup-axis')).val(); + case FRAME_EDIT_TYPE.SORT_VALUES: + let values = []; + $(this.wrapSelector('.vp-inner-popup-sortby-item')).each((idx, tag) => { + values.push($(tag).data('code')); }); + content['values'] = values; + content['ascending'] = $(this.wrapSelector('.vp-inner-popup-isascending')).val(); break; - // case FRAME_EDIT_TYPE.REPLACE: - // var useregex = $(this.wrapSelector('.vp-inner-popup-use-regex')).prop('checked'); - // content['useregex'] = useregex; - // content['list'] = []; - // for (var i=0; i <= this.state.popup.replace.index; i++) { - // var origin = $(this.wrapSelector('.vp-inner-popup-origin' + i)).val(); - // var origintext = $(this.wrapSelector('.vp-inner-popup-origin-istext'+i)).prop('checked'); - // var replace = $(this.wrapSelector('.vp-inner-popup-replace' + i)).val(); - // var replacetext = $(this.wrapSelector('.vp-inner-popup-replace-istext'+i)).prop('checked'); - // if (origin && replace) { - // content['list'].push({ - // origin: origin, - // origintext: origintext, - // replace: replace, - // replacetext: replacetext - // }); - // } - // } - // break; case FRAME_EDIT_TYPE.AS_TYPE: - this.state.selected.forEach((col, idx) => { + let selectedList = this.state.selected; + if (selectedList.length === 0) { + // select all + selectedList = this.state.columnList; + } + selectedList.forEach((col, idx) => { var value = $(this.wrapSelector('.vp-inner-popup-astype'+idx)).val(); - content[idx] = { - label: col.code, - value: value - }; + if (value !== undefined && value !== '') { + content[idx] = { + label: col.code, + value: value + }; + } }); break; + case FRAME_EDIT_TYPE.TO_DATETIME: + content['format'] = $(this.wrapSelector('.vp-inner-popup-todt-format')).val(); + content['format_typing'] = $(this.wrapSelector('.vp-inner-popup-todt-format-typing')).val(); + content['dayfirst'] = $(this.wrapSelector('.vp-inner-popup-todt-dayfirst')).val(); + content['newcol'] = $(this.wrapSelector('.vp-inner-popup-todt-new-col')).val(); + var colList = []; + var addcolItemTags = $(this.wrapSelector('.vp-inner-popup-todt-addcol-item')); + addcolItemTags && addcolItemTags.each((idx, tag) => { + let colName = $(tag).find('.vp-inner-popup-todt-addcol-colname').val(); + let dateType = $(tag).find('.vp-inner-popup-todt-addcol-type').val(); + if (colName !== '' && dateType !== '') { + colList.push({ colName: colName, dateType: dateType }); + } + }); + content['collist'] = colList; + break; + case FRAME_EDIT_TYPE.DISCRETIZE: + content['input'] = $(this.wrapSelector('.vp-inner-popup-input')).val(); + content['inputastext'] = $(this.wrapSelector('.vp-inner-popup-inputastext')).prop('checked'); + content['bins'] = $(this.wrapSelector('.vp-inner-popup-bins')).val(); + content['type'] = $(this.wrapSelector('.vp-inner-popup-discretizetype')).val(); + content['isright'] = $(this.wrapSelector('.vp-inner-popup-right')).prop('checked'); + let labelastext = $(this.wrapSelector('.vp-inner-popup-labelastext')).prop('checked'); + let islabelchanged = $(this.wrapSelector('.vp-inner-popup-islabelchanged')).val() === 'true'; + let isedgechanged = $(this.wrapSelector('.vp-inner-popup-isedgechanged')).val() === 'true'; + let rangeTableTags = $(this.wrapSelector('.vp-inner-popup-range-table tbody tr')); + let labels = []; + let edges = []; + rangeTableTags && rangeTableTags.each((idx, tag) => { + if (islabelchanged === true) { + labels.push(com_util.convertToStr($(tag).find('.vp-inner-popup-label').val(), labelastext)); + } + if (content['type'] === 'cut' && isedgechanged === true) { + edges.push($(tag).find('.vp-inner-popup-left-edge').val()); + if (idx === (rangeTableTags.length - 1)) { + edges.push($(tag).find('.vp-inner-popup-right-edge').val()); + } + } + }); + content['labels'] = labels; + content['edges'] = edges; + break; + case FRAME_EDIT_TYPE.DATA_SHIFT: + content['periods'] = $(this.wrapSelector('.vp-inner-popup-periods')).val(); + content['freq'] = $(this.wrapSelector('.vp-inner-popup-freq')).val(); + let fillValue = $(this.wrapSelector('.vp-inner-popup-fillvalue')).val(); + let fillValueAsText= $(this.wrapSelector('.vp-inner-popup-fillvalueastext')).prop('checked'); + content['fill_value'] = ''; + if (fillValue && fillValue !== '') { + content['fill_value'] = com_util.convertToStr(fillValue, fillValueAsText); + } + break; + case FRAME_EDIT_TYPE.FILL_NA: + content['method'] = $(this.wrapSelector('.vp-inner-popup-method')).val(); + content['value'] = $(this.wrapSelector('.vp-inner-popup-value')).val(); + content['valueastext'] = $(this.wrapSelector('.vp-inner-popup-valueastext')).prop('checked'); + content['limit'] = $(this.wrapSelector('.vp-inner-popup-limit')).val(); + break; + case FRAME_EDIT_TYPE.DROP_NA: + content['how'] = $(this.wrapSelector('.vp-inner-popup-how')).val(); + content['thresh'] = $(this.wrapSelector('.vp-inner-popup-thresh')).val(); + content['ignore_index'] = $(this.wrapSelector('.vp-inner-popup-ignoreindex')).val(); + break; + case FRAME_EDIT_TYPE.FILL_OUT: + content['filltype'] = $(this.wrapSelector('.vp-inner-popup-filltype')).val(); + content['fillvalue'] = $(this.wrapSelector('.vp-inner-popup-fillvalue')).val(); + break; + case FRAME_EDIT_TYPE.DROP_DUP: + content['keep'] = $(this.wrapSelector('.vp-inner-popup-how')).val(); + content['ignore_index'] = $(this.wrapSelector('.vp-inner-popup-ignoreindex')).val(); + break; default: break; } @@ -1134,7 +3074,7 @@ define([ renderInfoPage = function(renderedText, isHtml = true) { var tag = new com_String(); tag.appendFormatLine('

', VP_FE_INFO_CONTENT - , 'rendered_html'); // 'rendered_html' style from jupyter output area + , 'vp_rendered_html'); // 'rendered_html' style from jupyter output area if (isHtml) { tag.appendLine(renderedText); } else { @@ -1149,7 +3089,7 @@ define([ // get selected columns/indexes var selected = []; - $(this.wrapSelector('.' + VP_FE_TABLE + ' th.selected')).each((idx, tag) => { + $(this.wrapSelector(`.${VP_FE_TABLE} th:not(.${VP_FE_TABLE_COLUMN_GROUP}).selected`)).each((idx, tag) => { var label = $(tag).text(); var code = $(tag).data('code'); var type = $(tag).data('type'); @@ -1174,8 +3114,6 @@ define([ // code.append(".value_counts()"); code.appendFormat('_vp_display_dataframe_info({0})', locObj.toString()); - // CHROME: TODO: 6: use com_Kernel.execute - // Jupyter.notebook.kernel.execute( vpKernel.execute(code.toString()).then(function(resultObj) { let { msg } = resultObj; if (msg.content.data) { @@ -1197,10 +3135,10 @@ define([ } } else { var errorContent = ''; - if (msg.content.ename) { + if (msg.content && msg.content.ename) { errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue); } - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); $(that.wrapSelector('.' + VP_FE_INFO_CONTENT)).replaceWith(function() { return that.renderInfoPage(errorContent); }); @@ -1208,10 +3146,10 @@ define([ }).catch(function(resultObj) { let { msg } = resultObj; var errorContent = ''; - if (msg.content.ename) { - errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue); + if (msg.content && msg.content.ename) { + errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content?.evalue); } - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); $(that.wrapSelector('.' + VP_FE_INFO_CONTENT)).replaceWith(function() { return that.renderInfoPage(errorContent); }); @@ -1231,6 +3169,14 @@ define([ var selectedName = this.state.selected.map(col=>col.code).join(','); var axis = this.state.axis; + var subsetObjStr = tempObj; + if (selectedName && selectedName !== '') { + if (this.state.selected.length > 1) { + subsetObjStr += "[[" + selectedName + "]]"; + } else { + subsetObjStr += "[" + selectedName + "]"; + } + } var code = new com_String(); switch (type) { @@ -1243,43 +3189,83 @@ define([ break; case FRAME_EDIT_TYPE.RENAME: var renameList = []; - Object.keys(content).forEach((key, idx) => { - if (content[key].value != '') { - renameList.push(com_util.formatString("{0}: {1}", content[key].label, com_util.convertToStr(content[key].value, content[key].istext))); + Object.keys(content['list']).forEach((key, idx) => { + if (content['list'][key].value !== undefined && content['list'][key].value !== '') { + renameList.push(com_util.formatString("{0}: {1}", content['list'][key].label, com_util.convertToStr(content['list'][key].value, content['list'][key].istext))); } }); if (renameList.length > 0) { - code.appendFormat("{0}.rename({1}={{2}}, inplace=True)", tempObj, axis==FRAME_AXIS.ROW?'index':'columns', renameList.join(', ')); + code.appendFormat("{0}.rename({1}={{2}}", tempObj, axis==FRAME_AXIS.ROW?'index':'columns', renameList.join(', ')); + if (content['level'] !== undefined) { + code.appendFormat(", level={0}", content['level']); + } + code.append(', inplace=True)') } break; case FRAME_EDIT_TYPE.DROP_NA: - var locObj = ''; + var dropNAOptions = []; if (axis == FRAME_AXIS.ROW) { - locObj = com_util.formatString('.loc[[{0}],:]', selectedName); + dropNAOptions.push("axis=1"); } else { - locObj = com_util.formatString('.loc[:,[{0}]]', selectedName); + dropNAOptions.push("axis=0"); + } + if (selectedName && selectedName !== '') { + dropNAOptions.push(com_util.formatString("subset=[{0}]", selectedName)); + } + if (content.how && content.how !== '') { + dropNAOptions.push(com_util.formatString("how='{0}'", content.how)); } - code.appendFormat("{0}{1}.dropna(axis={2}, inplace=True)", tempObj, locObj, axis); + if (content.thresh && content.thresh !== '') { + dropNAOptions.push(com_util.formatString("thresh={0}", content.thresh)); + } + if (content.ignore_index && content.ignore_index !== '') { + dropNAOptions.push(com_util.formatString("ignore_index={0}", content.ignore_index)); + } + dropNAOptions.push("inplace=True"); + code.appendFormat("{0}.dropna({1})", tempObj, dropNAOptions.join(', ')); break; case FRAME_EDIT_TYPE.DROP_DUP: + let dropDupOptions = []; + if (selectedName && selectedName !== '') { + dropDupOptions.push(com_util.formatString("subset=[{0}]", selectedName)); + } + if (content.keep && content.keep !== '') { + dropDupOptions.push(com_util.formatString("keep={0}", content.keep)); + } + if (content.ignore_index && content.ignore_index !== '') { + dropDupOptions.push(com_util.formatString("ignore_index={0}", content.ignore_index)); + } + dropDupOptions.push("inplace=True"); + code.appendFormat("{0}.drop_duplicates({1})", tempObj, dropDupOptions.join(', ')); + break; + case FRAME_EDIT_TYPE.FILL_OUT: if (axis == FRAME_AXIS.COLUMN) { - code.appendFormat("{0}.drop_duplicates(subset=[{1}], inplace=True)", tempObj, selectedName); + code.appendFormat("{0} = vp_fill_outlier({1}, [{2}], fill_type='{3}'", tempObj, tempObj, selectedName, content.filltype); + if (content.filltype === 'value') { + code.appendFormat(", fill_value_lst={0}", content.fillvalue); + } + code.append(")"); } break; case FRAME_EDIT_TYPE.DROP_OUT: if (axis == FRAME_AXIS.COLUMN) { - code.appendFormat("{0} = vp_drop_outlier({1}, {2})", tempObj, tempObj, selectedName); + code.appendFormat("{0} = vp_drop_outlier({1}, [{2}])", tempObj, tempObj, selectedName); } break; case FRAME_EDIT_TYPE.LABEL_ENCODING: if (axis == FRAME_AXIS.COLUMN) { - let encodedColName = this.state.selected.map(col=> { + let encodedColNameList = this.state.selected.map(col=> { if (col.code !== col.label) { - return com_util.formatString("'{0}'", col.label + '_label'); + return { 'origin': com_util.formatString("'{0}'", col.label), 'encoded': com_util.formatString("'{0}'", col.label + '_label') }; } - return col.label + '_label' - }).join(','); - code.appendFormat("{0}[{1}] = pd.Categorical({2}[{3}]).codes", tempObj, encodedColName, tempObj, selectedName); + return { 'origin': col.label, 'encoded': col.label + '_label' }; + }); + encodedColNameList.forEach((encodedColObj, idx) => { + if (idx > 0) { + code.appendLine(); + } + code.appendFormat("{0}[{1}] = pd.Categorical({2}[{3}]).codes", tempObj, encodedColObj['encoded'], tempObj, encodedColObj['origin']); + }); } break; case FRAME_EDIT_TYPE.ONE_HOT_ENCODING: @@ -1293,30 +3279,170 @@ define([ } break; case FRAME_EDIT_TYPE.RESET_IDX: - if (axis == FRAME_AXIS.ROW) { - code.appendFormat("{0}.reset_index(inplace=True)", tempObj); + code.appendFormat("{0}.reset_index(inplace=True)", tempObj); + break; + case FRAME_EDIT_TYPE.SORT_INDEX: + let selectedStr = ''; + if (content.values.length > 1) { + selectedStr = content.values.join(','); + } + code.appendFormat("{0}.sort_index(axis={1}, ascending={2}", tempObj, content.axis, content.ascending); + if (selectedStr !== '') { + code.appendFormat(', level=[{0}]', selectedStr); + } + code.append(', inplace=True)'); + break; + case FRAME_EDIT_TYPE.SORT_VALUES: + if (axis == FRAME_AXIS.COLUMN) { + let selectedStr = ''; + if (content.values.length > 1) { + selectedStr = "[" + content.values.join(',') + "]"; + } else { + selectedStr = content.values[0]; + } + code.appendFormat("{0}.sort_values(by={1}, ascending={2}, inplace=True)", tempObj, selectedStr, content.ascending); } break; case FRAME_EDIT_TYPE.ADD_COL: - case FRAME_EDIT_TYPE.REPLACE: // if no name entered if (content.name == '') { return ''; } - var name = com_util.convertToStr(content.name, content.nameastext); - if (type == FRAME_EDIT_TYPE.REPLACE) { - name = selectedName; - } var tab = content.addtype; - if (tab == 'value') { - var value = com_util.convertToStr(content.value, content.valueastext); - code.appendFormat("{0}[{1}] = {2}", tempObj, name, value); - } else if (tab == 'calculation') { - var { var1col, oper, var2col } = content; - var var1code = tempObj + "['" + var1col + "']"; - var var2code = tempObj + "['" + var2col + "']"; - code.appendFormat('{0}[{1}] = {2} {3} {4}', tempObj, name, var1code, oper, var2code); + if (tab == 'calculate') { + let values = []; + content['values'] && content['values'].forEach((val, idx) => { + if (idx > 0) { + values.push(content['opers'][idx - 1]); + } + values.push(val); + }); + let valueStr = values.join(' '); + if (valueStr === "" || valueStr === "''") { + code.appendFormat("{0}[{1}] = np.nan", tempObj, content.name); + } else { + code.appendFormat("{0}[{1}] = {2}", tempObj, content.name, valueStr); + } + } else if (tab == 'statistics') { + code.appendFormat("{0}[{1}] = ", tempObj, content.name); + code.append(tempObj); + if (content['columns'].length > 0) { + code.appendFormat("[[{0}]]", content['columns'].join(',')); + } + code.appendFormat(".apply(lambda x: x.{0}(numeric_only=True), axis=1)", content.method); + // code.appendFormat(".{0}(axis=1)", content.method); // FIXME: for pandas version upper 2.0.0 } else if (tab == 'replace') { + var replaceStr = new com_String(); + var targetName = content['target']; + var useRegex = content['useregex']; + content['list'].forEach((obj, idx) => { + if (idx == 0) { + replaceStr.appendFormat("{0}: {1}" + , com_util.convertToStr(obj.origin, obj.origintext, useRegex) + , com_util.convertToStr(obj.replace, obj.replacetext, useRegex)); + } else { + replaceStr.appendFormat(", {0}: {1}" + , com_util.convertToStr(obj.origin, obj.origintext, useRegex) + , com_util.convertToStr(obj.replace, obj.replacetext, useRegex)); + } + }); + code.appendFormat("{0}[{1}] = {2}[[{3}]].replace({{4}}", tempObj, content.name, tempObj, targetName, replaceStr); + if (useRegex) { + code.append(', regex=True'); + } + code.append(')'); + } else if (tab == 'apply') { + // code.appendFormat("{0}[{1}] = {2}[{3}].apply({4})", tempObj, content.name, tempObj, content.column, content.apply); + let lambdaCode = 'lambda x: '; + content['caseList'].forEach(obj => { + // value if (cond list) else + let caseCode = obj.value + ' '; + let condCode = ''; + obj.condList.forEach((condObj, idx) => { + let { oper, cond, connector } = condObj; + if (oper === 'isnull()') { + condCode += `(pd.isnull(x))`; + } else if (oper === 'notnull()') { + condCode += `(pd.notnull(x))`; + } else if (oper === 'contains') { + condCode += `(${cond} in x)`; + } else if (oper === 'not contains') { + condCode += `(${cond} not in x)`; + } else if (oper === 'starts with') { + condCode += `(x.startswith(${cond}))`; + } else if (oper === 'ends with') { + condCode += `(x.endswith(${cond}))`; + } else { + condCode += `(x ${oper} ${cond})`; + } + if (connector !== undefined) { + condCode += ` ${connector} `; + } + }); + caseCode += 'if ' + condCode + ' else '; + lambdaCode += caseCode; + }); + lambdaCode += content['else']; + code.appendFormat("{0}[{1}] = {2}[{3}].apply({4})", tempObj, content.name, tempObj, content.target, lambdaCode); + } else if (tab === 'condition') { + code.appendFormat("{0}.loc[", tempObj); + content['list'].forEach((obj, idx) => { + let { colName, oper, cond, condAsText, connector } = obj; + code.append('('); + + let colValue = tempObj; + if (colName && colName != '') { + if (colName == '.index') { + colValue += colName; + } else { + colValue += com_util.formatString('[{0}]', colName); + } + } + let condValue = com_util.convertToStr(cond, condAsText); + if (oper == 'contains') { + code.appendFormat('{0}.str.contains({1})', colValue, condValue); + } else if (oper == 'not contains') { + code.appendFormat('~{0}.str.contains({1})', colValue, condValue); + } else if (oper == 'starts with') { + code.appendFormat('{0}.str.startswith({1})', colValue, condValue); + } else if (oper == 'ends with') { + code.appendFormat('{0}.str.endswith({1})', colValue, condValue); + } else if (oper == 'isnull()' || oper == 'notnull()') { + code.appendFormat('{0}.{1}', colValue, oper); + } else { + code.appendFormat('{0}{1}{2}', colValue, oper != ''?(' ' + oper):'', condValue != ''?(' ' + condValue):''); + } + code.append(')'); + if (idx < (content['list'].length - 1)) { + code.append(connector); + } + }); + var value = com_util.convertToStr(content.value, content.valueastext); + if (value === '') { + value = 'np.nan'; + } + code.appendFormat(", {0}] = {1}", content.name, value); + } + break; + case FRAME_EDIT_TYPE.ADD_ROW: + // if no name entered + if (content.name == '') { + return ''; + } + var tab = content.addtype; + let values = []; + content['values'] && content['values'].forEach((val, idx) => { + if (idx > 0) { + values.push(content['opers'][idx - 1]); + } + values.push(val); + }); + code.appendFormat("{0}.loc[{1}] = {2}", tempObj, content.name, values.join(' ')); + break; + case FRAME_EDIT_TYPE.REPLACE: + var name = content.name; + var tab = content.replacetype; + if (tab === 'replace') { var replaceStr = new com_String(); var useRegex = content['useregex']; content['list'].forEach((obj, idx) => { @@ -1338,18 +3464,83 @@ define([ code.append(', regex=True'); } code.append(')'); - } else if (tab == 'subset') { + } else if (tab === 'condition') { + code.appendFormat("{0}.loc[", tempObj); + var condCode = new com_String(); + content['list'].forEach((obj, idx) => { + let { colName, oper, cond, condAsText, connector } = obj; + condCode.append('('); + + let colValue = tempObj; + if (colName && colName != '') { + if (colName == '.index') { + colValue += colName; + } else { + colValue += com_util.formatString('[{0}]', colName); + } + } + let condValue = com_util.convertToStr(cond, condAsText); + if (oper == 'contains') { + condCode.appendFormat('{0}.str.contains({1})', colValue, condValue); + } else if (oper == 'not contains') { + condCode.appendFormat('~{0}.str.contains({1})', colValue, condValue); + } else if (oper == 'starts with') { + condCode.appendFormat('{0}.str.startswith({1})', colValue, condValue); + } else if (oper == 'ends with') { + condCode.appendFormat('{0}.str.endswith({1})', colValue, condValue); + } else if (oper == 'isnull()' || oper == 'notnull()') { + condCode.appendFormat('{0}.{1}', colValue, oper); + } else { + condCode.appendFormat('{0}{1}{2}', colValue, oper != ''?(' ' + oper):'', condValue != ''?(' ' + condValue):''); + } + condCode.append(')'); + if (idx < (content['list'].length - 1)) { + condCode.append(connector); + } + }); + if (condCode.toString() === '') { + condCode.append(':'); + } var value = com_util.convertToStr(content.value, content.valueastext); - code.appendFormat("{0} = {1}", content.subset, value); + if (value === '') { + value = 'np.nan'; + } + code.appendFormat("{0}, {1}] = {2}", condCode.toString(), content.name, value); } else if (tab == 'apply') { - code.appendFormat("{0}[{1}] = {2}[{3}].apply({4})", tempObj, name, tempObj, content.column, content.apply); + // code.appendFormat("{0}[{1}] = {2}[{3}].apply({4})", tempObj, content.name, tempObj, content.column, content.apply); + let lambdaCode = 'lambda x: '; + content['caseList'].forEach(obj => { + // value if (cond list) else + let caseCode = obj.value + ' '; + let condCode = ''; + obj.condList.forEach((condObj, idx) => { + let { oper, cond, connector } = condObj; + if (oper === 'isnull()') { + condCode += `pd.isnull(x)`; + } else if (oper === 'notnull()') { + condCode += `pd.notnull(x)`; + } else if (oper === 'contains') { + condCode += `(${cond} in x)`; + } else if (oper === 'not contains') { + condCode += `(${cond} not in x)`; + } else if (oper === 'starts with') { + condCode += `(x.startswith(${cond}))`; + } else if (oper === 'ends with') { + condCode += `(x.endswith(${cond}))`; + } else { + condCode += `(x ${oper} ${cond})`; + } + if (connector !== undefined) { + condCode += ` ${connector} `; + } + }); + caseCode += 'if ' + condCode + ' else '; + lambdaCode += caseCode; + }); + lambdaCode += content['else']; + code.appendFormat("{0}[{1}] = {2}[{3}].apply({4})", tempObj, content.name, tempObj, content.target, lambdaCode); } break; - case FRAME_EDIT_TYPE.ADD_ROW: - var name = com_util.convertToStr(content.name, content.nameastext); - var value = com_util.convertToStr(content.value, content.valueastext); - code.appendFormat("{0}.loc[{1}] = {2}", tempObj, name, value); - break; case FRAME_EDIT_TYPE.AS_TYPE: var astypeStr = new com_String(); Object.keys(content).forEach((key, idx) => { @@ -1361,6 +3552,83 @@ define([ }); code.appendFormat("{0} = {1}.astype({{2}})", tempObj, tempObj, astypeStr.toString()); break; + case FRAME_EDIT_TYPE.TO_DATETIME: + code.appendFormat("{0}['{1}'] = pd.to_datetime({2}[{3}]", tempObj, content['newcol'], tempObj, selectedName); + let optionList = []; + if (content['format'] === 'auto') { + if (content['dayfirst'] !== '') { + optionList.push(`dayfirst=${content['dayfirst']}`); + } + } else if (content['format'] === 'typing') { + if (content['format_typing'] !== '') { + optionList.push(`format='${content['format_typing']}'`); + } + } else { + optionList.push(`format='${content['format']}'`); + } + if (optionList.length > 0) { + code.appendFormat(', {0}', optionList.join(', ')); + } + code.append(')'); + if (content['collist'].length > 0) { + content['collist'].forEach(obj => { + code.appendLine(); + code.appendFormat("{0}['{1}'] = {2}['{3}'].dt.{4}", tempObj, obj.colName, tempObj, content['newcol'], obj.dateType); + }); + } + break; + case FRAME_EDIT_TYPE.DISCRETIZE: + let newColumn = com_util.convertToStr(content['input'], content['inputastext']); + let method = content['type']; + let bins = content['bins']; + if (method === 'cut') { + if (content['edges'] && content['edges'].length > 0) { + bins = "[" + content['edges'].join(',') + "]"; + } + } + + code.appendFormat("{0}[{1}] = pd.{2}({3}[{4}], {5}" + , tempObj, newColumn, method, tempObj, selectedName, bins); + + if (method === 'cut' && content['isright'] === false) { + code.append(", right=False"); + } + if (content['labels'] && content['labels'].length > 0) { + code.appendFormat(", labels=[{0}]", content['labels'].join(', ')); + } else { + code.append(", labels=False"); + } + code.append(')'); + break; + case FRAME_EDIT_TYPE.DATA_SHIFT: + code.appendFormat("{0} = {1}.shift({2}", subsetObjStr, subsetObjStr, content['periods']); + if (content['freq'] && content['freq'] !== '') { + code.appendFormat(", freq='{0}'", content['freq']); + } + if (content['fill_value'] && content['fill_value'] !== '') { + code.appendFormat(", fill_value={0}", content['fill_value']); + } + code.append(')'); + break; + case FRAME_EDIT_TYPE.FILL_NA: + code.appendFormat("{0} = {1}.fillna(", subsetObjStr, subsetObjStr); + if (content['method'] == 'value') { + code.append(com_util.convertToStr(content['value'], content['valueastext'])); + } else if (content['method'] === 'ffill' || content['method'] === 'bfill') { + code.appendFormat("method='{0}'", content['method']); + if (content['limit'] && content['limit'] !== '') { + code.appendFormat(", limit={0}", content['limit']); + } + } else { + if (content['method'] === 'mode') { + // get mode()'s first element (mode returns Series) + code.appendFormat("{0}.{1}()[0]", subsetObjStr, content['method']); + } else { + code.appendFormat("{0}.{1}()", subsetObjStr, content['method']); + } + } + code.append(')'); + break; case FRAME_EDIT_TYPE.SHOW: break; } @@ -1386,8 +3654,9 @@ define([ code.appendLine(codeStr); code.appendFormat("{0}.iloc[{1}:{2}].to_json(orient='{3}')", tempObj, prevLines, lines, 'split'); + this.loading = true; - vpKernel.execute(code.toString()).then(function(resultObj) { + vpKernel.execute(com_util.ignoreWarning(code.toString())).then(function(resultObj) { let { result } = resultObj; try { if (!result || result.length <= 0) { @@ -1401,12 +3670,13 @@ define([ vpKernel.getColumnList(tempObj).then(function(colResObj) { try { let columnResult = colResObj.result; - var columnList = JSON.parse(columnResult); + var columnInfo = JSON.parse(columnResult); + let { name:columnName='', level:columnLevel, list:columnList } = columnInfo; // var columnList = data.columns; var indexList = data.index; var dataList = data.data; - - columnList = columnList.map(col => { return { label: col.label, type: col.dtype, code: col.value } }); + + columnList = columnList.map(col => { return { label: col.label, type: col.dtype, code: col.value, isNumeric: col.is_numeric } }); indexList = indexList.map(idx => { return { label: idx, code: idx } }); if (!more) { @@ -1414,22 +3684,72 @@ define([ var table = new com_String(); // table.appendFormatLine('', 1, 'dataframe'); table.appendLine(''); - table.appendLine(''); - columnList && columnList.forEach(col => { - var colCode = col.code; - var colClass = ''; - if (that.state.axis == FRAME_AXIS.COLUMN && that.state.selected.map(col=>col.code).includes(colCode)) { - colClass = 'selected'; + if (columnLevel > 1) { + for (let colLevIdx = 0; colLevIdx < columnLevel; colLevIdx++) { + table.appendLine(''); + let colIdx = 0; + let colSpan = 1; + while (colIdx < columnList.length) { + let col = columnList[colIdx]; + let colCode = col.code.slice(0, colLevIdx + 1).join(','); + var colIcon = ''; + if (col.isNumeric === true) { + colIcon = ''; + } else { + colIcon = ''; + } + let nextCol = columnList[colIdx + 1]; + if (nextCol && nextCol.code.slice(0, colLevIdx + 1).join(',') === colCode) { + colSpan++; + } else { + let colClass = ''; + let selected = ''; // set class if it's leaf node of columns on multi-level + if (that.state.axis == FRAME_AXIS.COLUMN && that.state.selected.map(col=>col.code[colLevIdx]).includes(colCode)) { + selected = 'selected'; + } + if ((columnLevel - 1) === colLevIdx) { + colClass = VP_FE_TABLE_COLUMN; + } else { + colClass = VP_FE_TABLE_COLUMN_GROUP; + } + table.appendFormatLine('' + , colCode, FRAME_AXIS.COLUMN, col.type, col.label[colLevIdx-1], col.label[colLevIdx], colClass, selected, colSpan, colIcon, col.label[colLevIdx]); + colSpan = 1; + } + colIdx++; + } + + // add column + // LAB: img to url + // table.appendFormatLine('', VP_FE_ADD_COLUMN, com_Const.IMAGE_PATH + 'plus.svg'); + if (colLevIdx === 0) { + table.appendFormatLine('', VP_FE_ADD_COLUMN, 'vp-icon-plus'); + } + + table.appendLine(''); } - table.appendFormatLine('' - , colCode, FRAME_AXIS.COLUMN, col.type, VP_FE_TABLE_COLUMN, colClass, col.label); - }); - // add column - // LAB: img to url - // table.appendFormatLine('', VP_FE_ADD_COLUMN, com_Const.IMAGE_PATH + 'plus.svg'); - table.appendFormatLine('', VP_FE_ADD_COLUMN, 'vp-icon-plus'); - - table.appendLine(''); + } else { + table.appendLine(''); + columnList && columnList.forEach(col => { + var colCode = col.code; + var colIcon = ''; + if (col.isNumeric === true) { + colIcon = ''; + } else { + colIcon = ''; + } + var colClass = ''; + if (that.state.axis == FRAME_AXIS.COLUMN && that.state.selected.map(col=>col.code).includes(colCode)) { + colClass = 'selected'; + } + table.appendFormatLine('' + , colCode, FRAME_AXIS.COLUMN, col.type, col.label, VP_FE_TABLE_COLUMN, colClass, colIcon, col.label); + }); + // // add column + table.appendFormatLine('', VP_FE_ADD_COLUMN, 'vp-icon-plus'); + + table.appendLine(''); + } table.appendLine(''); table.appendLine(''); @@ -1460,7 +3780,7 @@ define([ table.appendLine(''); // LAB: img to url // table.appendFormatLine('', VP_FE_ADD_ROW, com_Const.IMAGE_PATH + 'plus.svg'); - table.appendFormatLine('', VP_FE_ADD_ROW, 'vp-icon-plus'); + // table.appendFormatLine('', VP_FE_ADD_ROW, 'vp-icon-plus'); table.appendLine(''); table.appendLine(''); $(that.wrapSelector('.' + VP_FE_TABLE)).replaceWith(function() { @@ -1496,6 +3816,7 @@ define([ } // save columnList & indexList as state + that.state.columnLevel = columnLevel; that.state.columnList = columnList; if (!more) { that.state.indexList = indexList; @@ -1506,6 +3827,8 @@ define([ // load info that.loadInfo(); + // load toolbar + that.renderToolbar(); // add to stack if (codeStr !== '') { let newSteps = codeStr.split('\n'); @@ -1516,6 +3839,11 @@ define([ var replacedCode = codeStr.replaceAll(that.state.tempObj, that.state.returnObj); that.setPreview(replacedCode); } + + // if add column, go to the right position + if (that.state.popup.type === FRAME_EDIT_TYPE.ADD_COL) { + $(that.wrapSelector('.vp-fe-add-column'))[0].scrollIntoView(); + } // if scrollPos is saved, go to the position if (scrollPos >= 0) { @@ -1534,11 +3862,21 @@ define([ that.loading = false; } }).catch(function(resultObj) { - let { result, type, msg } = resultObj; - vpLog.display(VP_LOG_TYPE.ERROR, result.ename + ': ' + result.evalue, msg, code.toString()); - if (that.isHidden() == false) { - // show alert modal only if this popup is visible - com_util.renderAlertModal(result.ename + ': ' + result.evalue); + let { result, msg, ename, evalue, status } = resultObj; + if (result) { + // NOTEBOOK: notebook error FIXME: migrate it on com_Kernel.execute + vpLog.display(VP_LOG_TYPE.ERROR, result?.ename + ': ' + result?.evalue, msg, code.toString()); + if (that.isHidden() == false) { + // show alert modal only if this popup is visible + com_util.renderAlertModal(result?.ename + ': ' + result?.evalue, { content: code.toString(), type: 'code' }); + } + } else { + // LAB: lab error FIXME: migrate it on com_Kernel.execute + vpLog.display(VP_LOG_TYPE.ERROR, ename + ': ' + evalue, status, code.toString()); + if (that.isHidden() == false) { + // show alert modal only if this popup is visible + com_util.renderAlertModal(ename + ': ' + evalue, { content: code.toString(), type: 'code' }); + } } that.loading = false; }); @@ -1601,6 +3939,7 @@ define([ const VP_FE_TABLE = 'vp-fe-table'; const VP_FE_TABLE_COLUMN = 'vp-fe-table-column'; + const VP_FE_TABLE_COLUMN_GROUP = 'vp-fe-table-column-group'; const VP_FE_TABLE_ROW = 'vp-fe-table-row'; const VP_FE_ADD_COLUMN = 'vp-fe-add-column'; const VP_FE_ADD_ROW = 'vp-fe-add-row'; @@ -1620,22 +3959,31 @@ define([ NONE: -1, INIT: 0, - RENAME: 2, + ADD_COL: 97, + ADD_ROW: 98, DROP: 3, - DROP_NA: 4, - DROP_DUP: 5, - DROP_OUT: 11, + RENAME: 2, + AS_TYPE: 10, + TO_DATETIME: 19, + REPLACE: 9, + DISCRETIZE: 15, + + SET_IDX: 7, + RESET_IDX: 8, + DATA_SHIFT: 14, + + SORT_INDEX: 16, + SORT_VALUES: 17, ONE_HOT_ENCODING: 6, LABEL_ENCODING: 12, - SET_IDX: 7, - RESET_IDX: 8, - REPLACE: 9, - AS_TYPE: 10, + FILL_NA: 13, + DROP_NA: 4, + FILL_OUT: 18, + DROP_OUT: 11, + DROP_DUP: 5, - ADD_COL: 97, - ADD_ROW: 98, SHOW: 99 } @@ -1645,5 +3993,11 @@ define([ COLUMN: 1 } + const FRAME_SELECT_TYPE = { + NONE: -1, // no problem with every selection type + SINGLE: 0, // only single select supported + MULTI: 1 // more than 1 selection needed + } + return Frame; }); \ No newline at end of file diff --git a/visualpython/js/m_apps/Groupby.js b/visualpython/js/m_apps/Groupby.js index 3636393e..589882f0 100644 --- a/visualpython/js/m_apps/Groupby.js +++ b/visualpython/js/m_apps/Groupby.js @@ -65,17 +65,17 @@ define([ this.methodList = [ { label: 'None', value: '' }, - { label: 'count', value: 'count' }, - { label: 'first', value: 'first' }, - { label: 'last', value: 'last' }, - { label: 'size', value: 'size' }, - { label: 'std', value: 'std' }, - { label: 'sum', value: 'sum' }, - { label: 'max', value: 'max' }, - { label: 'mean', value: 'mean' }, - { label: 'median', value: 'median' }, - { label: 'min', value: 'min' }, - { label: 'quantile', value: 'quantile' }, + { label: 'count', value: "count" }, + { label: 'first', value: "first" }, + { label: 'last', value: "last" }, + { label: 'size', value: "size" }, + { label: 'std', value: "std" }, + { label: 'sum', value: "sum" }, + { label: 'max', value: "max" }, + { label: 'mean', value: "mean" }, + { label: 'median', value: "median" }, + { label: 'min', value: "min" }, + { label: 'quantile', value: "quantile" }, ] this.state = { @@ -92,6 +92,8 @@ define([ advPageDom: '', advColList: [], + advMethodList: [], + advMethodUserList: [], advNamingList: [], ...this.state }; @@ -175,7 +177,7 @@ define([ var colList = event.dataList; that.state.display = colList; - if (colList && colList.length == 1) { + if ((colList && colList.length == 1) || that.state.method === 'size') { $(that.wrapSelector('#vp_gbToFrame')).parent().show(); } else { $(that.wrapSelector('#vp_gbToFrame')).parent().hide(); @@ -184,7 +186,9 @@ define([ // display select button event $(document).on('click', this.wrapSelector('#vp_gbDisplay'), function() { - that.openColumnSelector($(that.wrapSelector('#vp_gbDisplay')), 'Select columns to display'); + // exclude groupby columns + let excludeList = that.state.groupby.map(x => x.code); + that.openColumnSelector($(that.wrapSelector('#vp_gbDisplay')), 'Select columns to display', [], excludeList); }); // method select event @@ -192,6 +196,12 @@ define([ var method = $(this).val(); that.state.method = method; $(that.wrapSelector('#vp_gbMethod')).val(method); + + if (method === 'size' || (that.state.display && that.state.display.length == 1)) { + $(that.wrapSelector('#vp_gbToFrame')).parent().show(); + } else { + $(that.wrapSelector('#vp_gbToFrame')).parent().hide(); + } }); // advanced checkbox event @@ -276,33 +286,71 @@ define([ if (includeList && includeList.length > 0) { includeList = includeList.map(col => col.code); } - that.openColumnSelector($(this).parent().find('.vp-gb-adv-col'), 'Select columns', includeList); + // exclude groupby columns + let excludeList = that.state.groupby.map(x => x.code); + that.openColumnSelector($(this).parent().find('.vp-gb-adv-col'), 'Select columns', includeList, excludeList); }); // select method - $(document).on('change', this.wrapSelector('.vp-gb-adv-method-selector'), function() { - var method = $(this).val(); - var parentDiv = $(this).parent(); - if (method == 'typing') { - // change it to typing input - $(parentDiv).find('.vp-gb-adv-method-selector').hide(); - $(parentDiv).find('.vp-gb-adv-method').val(''); - $(parentDiv).find('.vp-gb-adv-method-box').show(); - } else { - $(parentDiv).find('.vp-gb-adv-method').val(com_util.formatString("'{0}'", method)); - } + $(document).on('click', this.wrapSelector('.vp-gb-adv-method'), function() { + // var method = $(this).val(); + // var parentDiv = $(this).parent(); + // if (method == 'typing') { + // // change it to typing input + // $(parentDiv).find('.vp-gb-adv-method-selector').hide(); + // $(parentDiv).find('.vp-gb-adv-method').val(''); + // $(parentDiv).find('.vp-gb-adv-method-box').show(); + // } else { + // $(parentDiv).find('.vp-gb-adv-method').val(com_util.formatString("'{0}'", method)); + // } + that.openMethodSelector($(this).parent().find('.vp-gb-adv-method'), 'Select methods'); }); // return to selecting method - $(document).on('click', this.wrapSelector('.vp-gb-adv-method-return'), function() { - var defaultValue = ''; - var parentDiv = $(this).parent().parent(); - $(parentDiv).find('.vp-gb-adv-method-selector').val(defaultValue); - $(parentDiv).find('.vp-gb-adv-method').val(defaultValue); - // show and hide - $(parentDiv).find('.vp-gb-adv-method-selector').show(); - $(parentDiv).find('.vp-gb-adv-method-box').hide(); + // $(document).on('click', this.wrapSelector('.vp-gb-adv-method-return'), function() { + // var defaultValue = ''; + // var parentDiv = $(this).parent().parent(); + // $(parentDiv).find('.vp-gb-adv-method-selector').val(defaultValue); + // $(parentDiv).find('.vp-gb-adv-method').val(defaultValue); + // // show and hide + // $(parentDiv).find('.vp-gb-adv-method-selector').show(); + // $(parentDiv).find('.vp-gb-adv-method-box').hide(); + // }); + // advanced item - method change event + $(document).on('change', this.wrapSelector('.vp-gb-adv-method'), function(event) { + var { list, userList } = event; + var idx = $(that.wrapSelector('.vp-gb-adv-method')).index(this); + that.state.advMethodList[idx] = list; + that.state.advMethodUserList[idx] = userList; + }); + + // advanced item - user method add event + $(document).on('click', this.wrapSelector('#vp_gbMethodUserAdd'), function(event) { + let userMethod = $(that.wrapSelector('#vp_gbMethodUser')).val(); + let useText = $(that.wrapSelector('#vp_gbMethodUserAsText')).prop('checked'); + let userCode = userMethod; + if (useText) { + userCode = "'" + userMethod + "'"; + } + that._addUserMethod(userCode); + }); + + // advanced item - user method add event by enter + $(document).on('keyup', this.wrapSelector('#vp_gbMethodUser'), function(event) { + var keycode = event.keyCode + ? event.keyCode + : event.which; + if (keycode == 13) { // enter + let userMethod = $(this).val(); + let useText = $(that.wrapSelector('#vp_gbMethodUserAsText')).prop('checked'); + let userCode = userMethod; + if (useText) { + userCode = "'" + userMethod + "'"; + } + that._addUserMethod(userCode); + } }); + // advanced item - naming change event $(document).on('change', this.wrapSelector('.vp-gb-adv-naming'), function(event) { @@ -318,8 +366,10 @@ define([ if (columns && columns.length > 0) { columns = columns.map(col => col.code); } - var method = $(parentDiv).find('.vp-gb-adv-method').val(); - if (!method || method == '' || method == "''") { + var method = $(parentDiv).find('.vp-gb-adv-method').data('list'); + var userMethod = $(parentDiv).find('.vp-gb-adv-method').data('userList'); + method = [ ...method, ...userMethod ]; + if (method == undefined || method.length <= 0) { // set focus on selecting method tag $(parentDiv).find('.vp-gb-adv-method').focus(); return; @@ -335,6 +385,20 @@ define([ }); } + _addUserMethod(userMethod) { + var addedMethodTag = $(this.wrapSelector(`.vp-gb-method-user-func[value="${userMethod}"]`)); + if (addedMethodTag.length > 0) { + // already added + addedMethodTag.prop('checked', true); + $(this.wrapSelector('#vp_gbMethodUser')).val(''); + return; + } + $(`${userMethod}`) + .appendTo($(this.wrapSelector('.vp-gb-method-user-box'))); + + $(this.wrapSelector('#vp_gbMethodUser')).val(''); + } + /** * Unbind events */ @@ -359,8 +423,9 @@ define([ $(document).off('click', this.wrapSelector('#vp_gbAdvAdd')); $(document).off('change', this.wrapSelector('.vp-gb-adv-col')); $(document).off('click', this.wrapSelector('.vp-gb-adv-col')); - $(document).off('change', this.wrapSelector('.vp-gb-adv-method-selector')); - $(document).off('click', this.wrapSelector('.vp-gb-adv-method-return')); + // $(document).off('change', this.wrapSelector('.vp-gb-adv-method-selector')); + $(document).off('click', this.wrapSelector('.vp-gb-adv-method')); + // $(document).off('click', this.wrapSelector('.vp-gb-adv-method-return')); $(document).off('change', this.wrapSelector('.vp-gb-adv-naming')); $(document).off('click', this.wrapSelector('.vp-gb-adv-naming')); $(document).off('click', this.wrapSelector('.vp-gb-adv-item-delete')); @@ -412,34 +477,31 @@ define([ page.appendFormatLine('

', 'vp-gb-adv-item'); // target columns page.appendFormatLine('' - , 'vp-gb-adv-col', 'Column list', 'Apply All columns, if not selected'); + , 'vp-gb-adv-col', 'All columns', 'Apply All columns, if not selected'); // method select - page.appendFormatLine(''); + // page.appendFormatLine(''); + page.appendFormatLine('' + , 'vp-gb-adv-method', 'Select methods to apply', ''); page.appendFormatLine(''); // naming - page.appendFormatLine('', 'vp-gb-adv-naming', 'Display name'); + page.appendFormatLine('', 'vp-gb-adv-naming', 'Click to rename columns', 'Click to rename columns'); // delete button - // LAB: img to url - // page.appendFormatLine('

', 'vp-gb-adv-item-delete', 'vp-cursor', com_Const.IMAGE_PATH + 'close_small.svg'); page.appendFormatLine('

', 'vp-gb-adv-item-delete', 'vp-cursor', 'vp-icon-close-small'); page.appendLine('

'); return page.toString(); @@ -458,7 +520,7 @@ define([ renderDataPage(renderedText, isHtml = true) { var tag = new com_String(); - tag.appendFormatLine('

', 'rendered_html'); // 'rendered_html' style from jupyter output area + tag.appendFormatLine('

', 'vp_rendered_html'); // 'rendered_html' style from jupyter output area if (isHtml) { tag.appendLine(renderedText); } else { @@ -487,20 +549,20 @@ define([ } } else { var errorContent = ''; - if (msg.content.ename) { - errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue); + if (msg.content && msg.content.ename) { + errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content?.evalue); } + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); that.renderDataPage(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); } }).catch(function(resultObj) { let { msg } = resultObj; var errorContent = ''; - if (msg.content.ename) { - errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue); + if (msg.content && msg.content.ename) { + errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content?.evalue); } + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); that.renderDataPage(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); }); } @@ -543,46 +605,100 @@ define([ * Render column selector using MultiSelector module * @param {Array} previousList previous selected columns * @param {Array} includeList columns to include + * @param {Array} excludeList columns to exclude */ - renderMultiSelector(previousList, includeList) { + renderMultiSelector(previousList, includeList, excludeList) { this.popup.ColSelector = new MultiSelector(this.wrapSelector('.vp-inner-popup-body'), - { mode: 'columns', parent: [this.state.variable], selectedList: previousList, includeList: includeList } + { mode: 'columns', parent: [this.state.variable], selectedList: previousList, includeList: includeList, excludeList: excludeList } ); } + /** + * Render method selector + * @param {Array} previousList previously selected methods ["'count'", ... ] + * @param {Array


	{8}{9}
{5}
	{6}{7}

{0}

', renderedText); + // More button + tag.appendFormatLine('

', VP_FE_TABLE_MORE, 'vp-button'); + } else { + tag.appendFormatLine('

{0}

', renderedText); + } + tag.appendLine('

'); // End of Table + return tag.toString(); + } + + generateCode() { + return this.tempCode; + } + + generateCodeForInfo() { + let { data, dtype, menu, menuItem } = this.state; + + var selected = []; + $(this.wrapSelector(`.${VP_FE_TABLE} th:not(.${VP_FE_TABLE_COLUMN_GROUP}).selected`)).each((idx, tag) => { + var label = $(tag).text(); + var code = $(tag).data('code'); + var type = $(tag).data('type'); + selected.push({ label: label, code: code, type: type }); + }); + this.state.selected = selected; + + let currentDtype = dtype; + let dataVar = new com_String(); + dataVar.append(data); + if (dtype === 'DataFrame') { + if (selected.length > 0) { + if (selected.length == 1) { + // series + dataVar.appendFormat("[{0}]", selected[0].code); + currentDtype = 'Series'; + } else { + // dataframe + dataVar.appendFormat("[[{0}]]", selected.map(col=>col.code).join(',')); + } + } + } + + if (menu && menu !== '') { + // get code pattern string + let codePattern = ''; + + // add info methods + let infoObj = this.menuList.find(obj=>obj.id === menu); + if (menu === 'statistics') { + // allow multi-select + if (menuItem && menuItem.length > 0) { + if (menuItem.length > 1) { + let statList = []; + menuItem && menuItem.forEach(itemId => { + let childObj = infoObj.child.find(obj=>obj.id === itemId); + statList.push(com_util.formatString("'{0}': {1}", itemId, childObj.code)); + }); + if (currentDtype === 'Series' && selected.length > 0) { + // if multiple stats selected, set series data as dataframe + dataVar = new com_String(); + dataVar.appendFormat("{0}[[{1}]]", data, selected.map(col=>col.code).join(',')); + currentDtype = 'DataFrame'; + } + codePattern = com_util.formatString("pd.DataFrame({{0}})", statList.join(',')); + } else { + if (currentDtype === 'Series' && selected.length > 0) { + // if multiple stats selected, set series data as dataframe + dataVar = new com_String(); + dataVar.appendFormat("{0}[[{1}]]", data, selected.map(col=>col.code).join(',')); + currentDtype = 'DataFrame'; + } + let childObj = infoObj.child.find(obj=>obj.id === menuItem[0]); + codePattern = childObj.code; + } + } else { + codePattern = infoObj.code; + } + } else { + // only one method selected + if (menuItem.length > 0 && infoObj.child) { + let childObj = infoObj.child.find(obj=>obj.id === menuItem[0]); + if (menuItem[0] === 'value_counts') { + if (currentDtype === 'Series') { + codePattern = childObj.code2; + } else { + codePattern = childObj.code; + } + } else { + if (childObj.toframe === true) { + if (dtype === 'Series') { + dataVar = new com_String(); + dataVar.appendFormat("{0}.to_frame()", data); + currentDtype = 'DataFrame'; + } else if (currentDtype === 'Series') { // DataFrame with single column selected + dataVar = new com_String(); + dataVar.appendFormat("{0}[[{1}]]", data, selected.map(col=>col.code).join(',')); + currentDtype = 'DataFrame'; + } + } + codePattern = childObj.code; + } + } else { + codePattern = infoObj.code; + } + } + + // match code pattern with data + if (codePattern) { + let code = codePattern.replaceAll('${data}', dataVar.toString()); + return code; + } + } + // else if (currentDtype == 'DataFrame') { + // // if dtype is dataframe, show describe + info preview + // let codePattern = "_desc = ${data}.describe().T\ + // \n_info = pd.DataFrame({'Non-Null Count': ${data}.notnull().sum(), 'Dtype': ${data}.dtypes})\ + // \ndisplay(pd.concat([_info, _desc], axis=1).fillna(''))"; + + // let code = codePattern.replaceAll('${data}', dataVar.toString()); + // return code; + // } + + // add ignore options TODO: + // let ignoreCode = `import warnings\ + // \nwith warnings.catch_warnings(): + // \n warnings.simplefilter("ignore")\n`; + + return `display(${dataVar.toString()})`; + } + + loadCode(codeStr, more=false) { + if (this.loading === true) { + return; + } + + var that = this; + let { data, lines, indexList } = this.state; + var prevLines = 0; + var scrollPos = -1; + if (more) { + prevLines = indexList.length; + scrollPos = $(this.wrapSelector('.vp-variable-table')).scrollTop(); + } + + var code = new com_String(); + code.appendLine(codeStr); + code.appendFormat("{0}.iloc[{1}:{2}].to_json(orient='{3}')", data, prevLines, lines, 'split'); + + this.loading = true; + vpKernel.execute(code.toString()).then(function(resultObj) { + let { result } = resultObj; + try { + if (!result || result.length <= 0) { + return; + } + result = result.substr(1,result.length - 2).replaceAll('\\\\', '\\'); + result = result.replaceAll('\'', "\\'"); // TEST: need test + // result = result.replaceAll('\\"', "\""); + var dataJson = JSON.parse(result); + + vpKernel.getColumnList(data).then(function(colResObj) { + try { + let columnResult = colResObj.result; + var columnInfo = JSON.parse(columnResult); + let { name:columnName='', level:columnLevel, list:columnList } = columnInfo; + // var columnList = data.columns; + var indexList = dataJson.index; + var dataList = dataJson.data; + + columnList = columnList.map(col => { return { label: col.label, type: col.dtype, code: col.value } }); + indexList = indexList.map(idx => { return { label: idx, code: idx } }); + + if (!more) { + // table + var table = new com_String(); + // table.appendFormatLine('', 1, 'dataframe'); + table.appendLine(''); + if (columnLevel > 1) { + for (let colLevIdx = 0; colLevIdx < columnLevel; colLevIdx++) { + table.appendLine(''); + let colIdx = 0; + let colSpan = 1; + while (colIdx < columnList.length) { + let col = columnList[colIdx]; + let colCode = col.code.slice(0, colLevIdx + 1).join(','); + let nextCol = columnList[colIdx + 1]; + if (nextCol && nextCol.code.slice(0, colLevIdx + 1).join(',') === colCode) { + colSpan++; + } else { + let colClass = ''; + let selected = ''; // set class if it's leaf node of columns on multi-level + if (that.state.selected.map(col=>col.code[colLevIdx]).includes(colCode)) { + selected = 'selected'; + } + if ((columnLevel - 1) === colLevIdx) { + colClass = VP_FE_TABLE_COLUMN; + } else { + colClass = VP_FE_TABLE_COLUMN_GROUP; + } + table.appendFormatLine('' + , colCode, FRAME_AXIS.COLUMN, col.type, col.label[colLevIdx-1], col.label[colLevIdx], colClass, selected, colSpan, col.label[colLevIdx]); + colSpan = 1; + } + colIdx++; + } + + table.appendLine(''); + } + } else { + table.appendLine(''); + columnList && columnList.forEach(col => { + var colCode = col.code; + var colClass = ''; + if (that.state.selected.map(col=>col.code).includes(colCode)) { + colClass = 'selected'; + } + table.appendFormatLine('' + , colCode, FRAME_AXIS.COLUMN, col.type, col.label, VP_FE_TABLE_COLUMN, colClass, col.label); + }); + table.appendLine(''); + } + table.appendLine(''); + table.appendLine(''); + + dataList && dataList.forEach((row, idx) => { + table.appendLine(''); + var idxName = indexList[idx].label; + var idxLabel = com_util.convertToStr(idxName, typeof idxName == 'string'); + var idxClass = ''; + table.appendFormatLine('', idxLabel, FRAME_AXIS.ROW, VP_FE_TABLE_ROW, idxClass, idxName); + row.forEach((cell, colIdx) => { + if (cell == null) { + cell = 'NaN'; + } + var cellType = columnList[colIdx].type; + if (cellType.includes('datetime')) { + cell = new Date(parseInt(cell)).toLocaleString(); + } + table.appendFormatLine('', cell); + }); + table.appendLine(''); + }); + // add row + table.appendLine(''); + table.appendLine(''); + table.appendLine(''); + $(that.wrapSelector('.' + VP_FE_TABLE)).replaceWith(function() { + return that.renderTable(table.toString()); + }); + } else { + var table = new com_String(); + dataList && dataList.forEach((row, idx) => { + table.appendLine(''); + var idxName = indexList[idx].label; + var idxLabel = com_util.convertToStr(idxName, typeof idxName == 'string'); + var idxClass = ''; + table.appendFormatLine('', idxLabel, FRAME_AXIS.ROW, VP_FE_TABLE_ROW, idxClass, idxName); + row.forEach((cell, colIdx) => { + if (cell == null) { + cell = 'NaN'; + } + var cellType = columnList[colIdx].type; + if (cellType.includes('datetime')) { + cell = new Date(parseInt(cell)).toLocaleString(); + } + table.appendFormatLine('', cell); + }); + table.appendLine(''); + }); + // insert before last tr tag(add row button) + $(table.toString()).insertBefore($(that.wrapSelector('.' + VP_FE_TABLE + ' tbody tr:last'))); + } + + // save columnList & indexList as state + that.state.columnLevel = columnLevel; + that.state.columnList = columnList; + if (!more) { + that.state.indexList = indexList; + } else { + that.state.indexList = that.state.indexList.concat(indexList); + } + + // if scrollPos is saved, go to the position + if (scrollPos >= 0) { + $(that.wrapSelector('.vp-variable-table')).scrollTop(scrollPos); + } + + that.loading = false; + } catch (err1) { + vpLog.display(VP_LOG_TYPE.ERROR, err1); + that.loading = false; + throw err1; + } + }); + } catch (err) { + vpLog.display(VP_LOG_TYPE.ERROR, err); + that.loading = false; + } + }).catch(function(resultObj) { + let { msg } = resultObj; + var errorContent = ''; + if (msg.content && msg.content.ename) { + errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content?.evalue, msg.content?.detail); + } + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); + $(variablePreviewTag).html(errorContent); + that.loading = false; + }); + + return code.toString(); + } + + /** + * Load information preview + */ + loadInfo() { + let that = this; + let { menu, menuItem } = this.state; + + $(this.wrapSelector('#informationPreviewTitle')).text(''); + if (menu) { + let { label, child } = this.menuList.find(x => x.id === menu); + let menuItemLabelList = child.filter(x => menuItem.includes(x.id)).map(x => x.label); + + // load preview title + $(this.wrapSelector('#informationPreviewTitle')).text(com_util.formatString('{0} > {1}', label, menuItemLabelList.join(', '))); + } + // load preview content + let $infoPreviewTag = $(this.wrapSelector('#informationPreview')); + $infoPreviewTag.html(''); + let code = this.generateCodeForInfo(); + this.tempCode = code; + + // use default pandas option + // let defaultPOCode = new com_String(); + // defaultPOCode.appendLine("with pd.option_context('display.min_rows', 10, 'display.max_rows', 60, 'display.max_columns', 0, 'display.max_colwidth', 50, 'display.expand_frame_repr', True, 'display.precision', 6, 'display.chop_threshold', None):"); + // defaultPOCode.append(' ' + code.replaceAll('\n', '\n ')); + + let loadingSpinner = new LoadingSpinner($(this.wrapSelector('#informationPreview'))); + // show variable information on clicking variable + vpKernel.execute(code).then(function(resultObj) { + let { result, type, msg } = resultObj; + if (type !== 'error' && result) { + $infoPreviewTag.html(''); + if (type == 'text/html') { + // 1. HTML tag + $infoPreviewTag.append(result); + } else if (type == 'image/png') { + // 2. Image data (base64) + var imgTag = '

'; + $infoPreviewTag.append(imgTag); + } else if (type == 'text/plain' || type == 'text') { + // 3. Text data + var preTag = document.createElement('pre'); + $(preTag).text(result); + $infoPreviewTag.html(preTag); + } else { + $infoPreviewTag.append('(Select data to preview.)'); + } + } else { + var errorContent = ''; + if (msg.content && msg.content.ename) { + errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue, msg.content.detail); + } + $infoPreviewTag.html(errorContent); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); + } + }).catch(function(resultObj) { + let { msg, ename, evalue } = resultObj; + var errorContent = ''; + if (msg && msg?.content?.ename) { + errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue, msg.content.detail); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); + } else if (ename && evalue) { + errorContent = com_util.templateForErrorBox(ename, evalue); + vpLog.display(VP_LOG_TYPE.ERROR, ename, evalue, resultObj); + } + $infoPreviewTag.html(errorContent); + }).finally(function() { + loadingSpinner.remove(); + }); + } + } + + // search rows count at once + const TABLE_LINES = 10; + + const VP_FE_TABLE = 'vp-variable-table'; + const VP_FE_TABLE_COLUMN = 'vp-variable-table-column'; + const VP_FE_TABLE_COLUMN_GROUP = 'vp-variable-table-column-group'; + const VP_FE_TABLE_ROW = 'vp-variable-table-row'; + const VP_FE_TABLE_MORE = 'vp-variable-table-more'; + + const FRAME_AXIS = { + NONE: -1, + ROW: 0, + COLUMN: 1 + } + + return Information; +}); \ No newline at end of file diff --git a/visualpython/js/m_apps/Instance.js b/visualpython/js/m_apps/Instance.js index a4f0e6c5..1095ef31 100644 --- a/visualpython/js/m_apps/Instance.js +++ b/visualpython/js/m_apps/Instance.js @@ -16,10 +16,12 @@ define([ __VP_TEXT_LOADER__('vp_base/html/m_apps/instance.html'), // INTEGRATION: unified version of text loader __VP_CSS_LOADER__('vp_base/css/m_apps/instance'), // INTEGRATION: unified version of css loader 'vp_base/js/com/com_String', + 'vp_base/js/com/com_util', 'vp_base/js/com/component/PopupComponent', 'vp_base/js/com/component/InstanceEditor', + 'vp_base/js/com/component/DataSelector', 'vp_base/js/m_apps/Subset' -], function(insHtml, insCss, com_String, PopupComponent, InstanceEditor, Subset) { +], function(insHtml, insCss, com_String, com_util, PopupComponent, InstanceEditor, DataSelector, Subset) { const MAX_STACK_SIZE = 20; @@ -31,10 +33,11 @@ define([ super._init(); /** Write codes executed before rendering */ this.config.dataview = false; - this.config.sizeLevel = 1; + this.config.sizeLevel = 5; this.config.checkModules = ['pd']; this.state = { + target: '', vp_instanceVariable: '', variable: { stack: [] @@ -53,6 +56,12 @@ define([ _bindEvent() { super._bindEvent(); let that = this; + // target change + $(this.wrapSelector('#vp_instanceTarget')).on('change', function(event) { + let value = $(this).val(); + that.updateValue(value); + that.reloadInsEditor(); + }); // clear $(this.wrapSelector('#vp_instanceClear')).on('click', function(event) { that.addStack(); @@ -139,7 +148,7 @@ define([ var selectedVariable = event.varName; let fullCode = nowCode + selectedVariable; that.updateValue(fullCode); - that.reloadInsEditor('variable'); + that.reloadInsEditor(); }); // instance_editor_replaced - variable @@ -148,7 +157,7 @@ define([ var newCode = event.newCode; that.updateValue(newCode); - that.reloadInsEditor('variable'); + that.reloadInsEditor(); }); // co-op with Subset @@ -176,8 +185,35 @@ define([ } templateForBody() { + let that = this; let page = $(insHtml); $(page).find('#vp_instanceVariable').val(this.state.vp_instanceVariable); + + let targetSelector = new DataSelector({ + pageThis: this, id: 'vp_instanceTarget', placeholder: 'Select variable', + allowDataType: [ + 'module', 'DataFrame', 'Series', 'dict', 'list', 'int' + ], + allowModule: true, + finish: function(value, dtype) { + $(that.wrapSelector('#vp_instanceTarget')).trigger({type: 'change', value: value}); + that.updateValue(value); + that.reloadInsEditor(); + }, + select: function(value, dtype) { + $(that.wrapSelector('#vp_instanceTarget')).trigger({type: 'change', value: value}); + that.updateValue(value); + that.reloadInsEditor(); + } + }); + $(page).find('#vp_instanceTarget').replaceWith(targetSelector.toTagString()); + + // Removed dataselector for Allocation input + // let allocateSelector = new DataSelector({ + // pageThis: this, id: 'vp_instanceAllocate', placeholder: 'Variable name' + // }); + // $(page).find('#vp_instanceAllocate').replaceWith(allocateSelector.toTagString()); + return page; } @@ -187,7 +223,7 @@ define([ let that = this; // vpSubsetEditor - this.subsetEditor = new Subset({ pandasObject: '', config: { name: 'Subset' } }, + this.subsetEditor = new Subset({ pandasObject: '', config: { name: 'Subset', category: this.name } }, { useInputVariable: true, targetSelector: this.wrapSelector('#vp_instanceVariable'), @@ -203,7 +239,7 @@ define([ this.ALLOW_SUBSET_TYPES = this.subsetEditor.getAllowSubsetTypes(); // vpInstanceEditor - this.insEditor = new InstanceEditor(this, "vp_instanceVariable", 'vp_variableInsEditContainer'); + this.insEditor = new InstanceEditor(this, "vp_instanceVariable", 'vp_variableInsEditContainer', { targetType: 'outside' }); this.insEditor.show(); @@ -227,7 +263,8 @@ define([ let cmObj = this.getCodemirror('vp_instanceVariable'); let rightCode = (cmObj && cmObj.cm)?cmObj.cm.getValue():''; if (leftCode && leftCode != '') { - sbCode.appendFormat('{0} = {1}', leftCode, rightCode); + sbCode.appendFormatLine('{0} = {1}', leftCode, rightCode); + sbCode.append(leftCode); // show allocation (from version 2.4.10) } else { sbCode.appendFormat('{0}', rightCode); } @@ -260,6 +297,61 @@ define([ cm.setCursor({ line: 0, ch: value.length}); } this.state.vp_instanceVariable = value; + + // show preview + this.loadPreview(value); + } + + loadPreview(code) { + let that = this; + if (!code || code === '') { + $(that.wrapSelector('#instancePreview')).html(''); + return; + } + // show variable information on clicking variable + vpKernel.execute(code).then(function(resultObj) { + let { result, type, msg } = resultObj; + if (msg.content.data) { + var textResult = msg.content.data["text/plain"]; + var htmlResult = msg.content.data["text/html"]; + var imgResult = msg.content.data["image/png"]; + + $(that.wrapSelector('#instancePreview')).html(''); + if (htmlResult != undefined) { + // 1. HTML tag + $(that.wrapSelector('#instancePreview')).append(htmlResult); + } else if (imgResult != undefined) { + // 2. Image data (base64) + var imgTag = '

'; + $(that.wrapSelector('#instancePreview')).append(imgTag); + } else if (textResult != undefined) { + // 3. Text data + var preTag = document.createElement('pre'); + $(preTag).text(textResult); + $(that.wrapSelector('#instancePreview')).html(preTag); + } + } else { + var errorContent = ''; + if (msg.content && msg.content.ename) { + errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue, msg.content.detail); + } + $(that.wrapSelector('#instancePreview')).html(errorContent); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); + } + }).catch(function(resultObj) { + let { msg, ename, evalue, status } = resultObj; + var errorContent = ''; + if (msg && msg.content && msg.content.ename) { + // NOTEBOOK: notebook error FIXME: migrate it on com_Kernel.execute + errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue, msg.content.detail); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); + } else { + // LAB: lab error FIXME: migrate it on com_Kernel.execute + errorContent = com_util.templateForErrorBox(ename, evalue, ''); + vpLog.display(VP_LOG_TYPE.ERROR, ename, evalue); + } + $(that.wrapSelector('#instancePreview')).html(errorContent); + }); } addStack() { @@ -286,9 +378,10 @@ define([ return lastValue; } - reloadInsEditor(type='') { + reloadInsEditor() { var that = this; var tempPointer = this.pointer; + this.subsetEditor.disableButton(); var callbackFunction = function (varObj) { var varType = varObj.type; @@ -298,8 +391,6 @@ define([ let nowCode = (cmObj && cmObj.cm)?cmObj.cm.getValue():''; that.subsetEditor.state.pandasObject = nowCode; that.subsetEditor.enableButton(); - } else { - that.subsetEditor.disableButton(); } }; diff --git a/visualpython/js/m_apps/Markdown.js b/visualpython/js/m_apps/Markdown.js index bfd2db48..ccceec89 100644 --- a/visualpython/js/m_apps/Markdown.js +++ b/visualpython/js/m_apps/Markdown.js @@ -202,7 +202,7 @@ define([ // CHROME: TODO: 4: marked is not loaded, before fix it comment it if (vpConfig.extensionType === 'notebook') { - var marked = require('marked'); + var marked = window.require('components/marked/lib/marked'); var renderer = new marked.Renderer(); // get block @@ -221,9 +221,8 @@ define([ MathJax.Hub.Queue(["Typeset", MathJax.Hub, "vp_markdownPreview"]); }); - } else if (vpConfig.extensionType === 'lab') { + } else if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { var marked = require('marked'); - var renderer = new marked.Renderer(); // get block let block = this.getTaskType() == 'block'? this.taskItem: null; diff --git a/visualpython/js/m_apps/PDF.js b/visualpython/js/m_apps/PDF.js index c4ea125b..e5fdf2ce 100644 --- a/visualpython/js/m_apps/PDF.js +++ b/visualpython/js/m_apps/PDF.js @@ -21,9 +21,9 @@ define([ 'vp_base/js/com/component/FileNavigation' ], function(pdfHtml, pdfCss, com_String, com_interface, PopupComponent, FileNavigation) { - const PDF_SHOW = '!pip show PyMuPDF nltk' - const PDF_INSTALL1 = '!pip install PyMuPDF' - const PDF_INSTALL2 = '!pip install nltk' + var PDF_SHOW = '!pip show PyMuPDF nltk' + var PDF_INSTALL1 = '!pip install PyMuPDF' + var PDF_INSTALL2 = '!pip install nltk' const PDF_IMPORT = `import pandas as pd import fitz @@ -80,6 +80,12 @@ nltk.download('punkt')`; vp_pdfReturn: '', ...this.state } + + if (vpConfig.extensionType === 'lite') { + PDF_SHOW = PDF_SHOW.replace('!', '%'); + PDF_INSTALL1 = PDF_INSTALL1.replace('!', '%'); + PDF_INSTALL2 = PDF_INSTALL2.replace('!', '%'); + } } _bindEvent() { diff --git a/visualpython/js/m_apps/PandasOption.js b/visualpython/js/m_apps/PandasOption.js new file mode 100644 index 00000000..359de409 --- /dev/null +++ b/visualpython/js/m_apps/PandasOption.js @@ -0,0 +1,150 @@ +/* + * Project Name : Visual Python + * Description : GUI-based Python code generator + * File Name : PandasOption.js + * Author : Black Logic + * Note : Pandas option + * License : GNU GPLv3 with Visual Python special exception + * Date : 2023. 03. 28 + * Change Date : + */ + +//============================================================================ +// [CLASS] PandasOption +//============================================================================ +define([ + __VP_TEXT_LOADER__('vp_base/html/m_apps/pandasOption.html'), + 'vp_base/js/com/com_util', + 'vp_base/js/com/com_Const', + 'vp_base/js/com/com_String', + 'vp_base/js/com/component/PopupComponent' +], function(poHTML, com_util, com_Const, com_String, PopupComponent) { + + /** + * PandasOption + */ + class PandasOption extends PopupComponent { + _init() { + super._init(); + /** Write codes executed before rendering */ + this.config.sizeLevel = 2; + this.config.dataview = false; + this.config.checkModules = ['pd']; + this.config.docs = 'https://pandas.pydata.org/docs/user_guide/options.html'; + + this.state = { + filter_warning: '', + min_rows: '', + max_rows: '', + max_columns: '', + max_colwidth: '', + expand_frame_repr: '', + precision: '', + chop_threshold: '', + ...this.state + } + } + + _bindEvent() { + super._bindEvent(); + /** Implement binding events */ + var that = this; + + // setting popup - set default + $(this.wrapSelector('#resetDisplay')).on('change', function() { + let checked = $(this).prop('checked'); + + if (checked) { + // disable input + $(that.wrapSelector('.vp-po-option-item')).prop('disabled', true); + } else { + // enable input + $(that.wrapSelector('.vp-po-option-item')).prop('disabled', false); + } + }); + + // show all rows + $(this.wrapSelector('#showAllRows')).on('change', function() { + let checked = $(this).prop('checked'); + if (checked) { + $(that.wrapSelector('#max_rows')).prop('disabled', true); + } else { + $(that.wrapSelector('#max_rows')).prop('disabled', false); + } + }); + + // show all columns + $(this.wrapSelector('#showAllColumns')).on('change', function() { + let checked = $(this).prop('checked'); + if (checked) { + $(that.wrapSelector('#max_columns')).prop('disabled', true); + } else { + $(that.wrapSelector('#max_columns')).prop('disabled', false); + } + }); + + // setting popup - reset warning + $(this.wrapSelector('#resetWarning')).on('change', function() { + let checked = $(this).prop('checked'); + + if (checked) { + // disable input + $(that.wrapSelector('.vp-po-warning-item')).prop('disabled', true); + } else { + // enable input + $(that.wrapSelector('.vp-po-warning-item')).prop('disabled', false); + } + }); + } + + templateForBody() { + let page = $(poHTML); + + return page; + } + + render() { + super.render(); + + + } + + generateCode() { + let that = this; + let code = []; + + let resetDisplay = $(this.wrapSelector('#resetDisplay')).prop('checked'); + let resetWarning = $(this.wrapSelector('#resetWarning')).prop('checked'); + + // warning options + if (resetWarning === true) { + code.push("import warnings\nwarnings.resetwarnings()"); + } else{ + if (that.state['filter_warning'] && that.state['filter_warning'] != '') { + code.push(com_util.formatString("import warnings\nwarnings.simplefilter(action='{0}', category=Warning)", that.state['filter_warning'])); + } + } + + // display options + if (resetDisplay === true) { + code.push("pd.reset_option('^display')"); + } else { + let showAllRows = $(this.wrapSelector('#showAllRows')).prop('checked'); + let showAllCols = $(this.wrapSelector('#showAllColumns')).prop('checked'); + Object.keys(this.state).forEach((key) => { + if ((showAllRows === true && key === 'max_rows') + || (showAllCols === true && key === 'max_columns')) { + code.push(com_util.formatString("pd.set_option('display.{0}', {1})", key, 'None')); + } else if (key !== 'filter_warning' && that.state[key] && that.state[key] != '') { + code.push(com_util.formatString("pd.set_option('display.{0}', {1})", key, that.state[key])); + } + }); + } + + return code.join('\n'); + } + + } + + return PandasOption; +}); \ No newline at end of file diff --git a/visualpython/js/m_apps/Profiling.js b/visualpython/js/m_apps/Profiling.js index 019d1ef2..e1df8df3 100644 --- a/visualpython/js/m_apps/Profiling.js +++ b/visualpython/js/m_apps/Profiling.js @@ -9,6 +9,24 @@ * Change Date : */ +/** DEPRECATED on v2.3.4 + * - libraries.json + { + "id" : "apps_profiling", + "type" : "function", + "level": 1, + "name" : "Profiling", + "tag" : "PROFILING,PANDAS PROFILING,APPS", + "path" : "visualpython - apps - profiling", + "desc" : "Pandas Profiling", + "file" : "m_apps/Profiling", + "apps" : { + "color": 4, + "icon": "apps/apps_profiling.svg" + } + } + */ + //============================================================================ // [CLASS] Profiling //============================================================================ diff --git a/visualpython/js/m_apps/Reshape.js b/visualpython/js/m_apps/Reshape.js index 8628ac46..995ddd1e 100644 --- a/visualpython/js/m_apps/Reshape.js +++ b/visualpython/js/m_apps/Reshape.js @@ -219,7 +219,9 @@ define([ // reset index checkbox event $(document).on('change', this.wrapSelector('#vp_rsResetIndex'), function() { - that.state.resetIndex = $(this).prop('checked'); + let isChecked = $(this).prop('checked'); + $(that.wrapSelector('#vp_rsWithoutColumn')).prop('disabled', !isChecked); + that.state.resetIndex = isChecked; }); // with/without column select event @@ -246,7 +248,7 @@ define([ renderDataPage(renderedText, isHtml = true) { var tag = new com_String(); - tag.appendFormatLine('

', 'rendered_html'); // 'rendered_html' style from jupyter output area + tag.appendFormatLine('

', 'vp_rendered_html'); // 'rendered_html' style from jupyter output area if (isHtml) { tag.appendLine(renderedText); } else { @@ -275,7 +277,7 @@ define([ } } else { var errorContent = ''; - if (msg.content.ename) { + if (msg.content && msg.content.ename) { errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue); } that.renderDataPage(errorContent); @@ -283,7 +285,7 @@ define([ }).catch(function(resultObj) { let { msg } = resultObj; var errorContent = ''; - if (msg.content.ename) { + if (msg.content && msg.content.ename) { errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue); } that.renderDataPage(errorContent); diff --git a/visualpython/js/m_apps/Snippets.js b/visualpython/js/m_apps/Snippets.js index c9656b84..1fecfce7 100644 --- a/visualpython/js/m_apps/Snippets.js +++ b/visualpython/js/m_apps/Snippets.js @@ -18,10 +18,11 @@ define([ 'vp_base/js/com/com_util', 'vp_base/js/com/com_Const', 'vp_base/js/com/com_String', + 'vp_base/js/com/com_interface', 'vp_base/js/com/component/PopupComponent', 'vp_base/js/com/component/FileNavigation', 'vp_base/js/com/component/LoadingSpinner' -], function(snHtml, snCss, com_util, com_Const, com_String, PopupComponent, FileNavigation, LoadingSpinner) { +], function(snHtml, snCss, com_util, com_Const, com_String, com_interface, PopupComponent, FileNavigation, LoadingSpinner) { /** * Snippets @@ -126,7 +127,41 @@ define([ filesPath.forEach(fileObj => { var fileName = fileObj.file; var selectedPath = fileObj.path; - fetch(selectedPath).then(function(file) { + if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { + vpKernel.readFile(selectedPath).then(function(resultObj) { + try { + var snippetData = JSON.parse(resultObj.result); + var timestamp = new Date().getTime(); + + var keys = Object.keys(snippetData); + var importKeys = []; + var newSnippet = {}; + keys.forEach(key => { + var importKey = key; + var importNo = 1; + var titleList = Object.keys(that.codemirrorList); + // set duplicate title + while(titleList.includes(importKey)) { + importKey = key + '_imported' + importNo; + importNo += 1; + } + newSnippet = { ...newSnippet, [importKey]: { code: snippetData[key], timestamp: timestamp } }; + + importKeys.push(importKey); + }); + vpConfig.setData(newSnippet).then(function() { + that.importedList = [ ...importKeys ]; + that.loadUdfList(); + com_util.renderSuccessMessage(fileName + ' imported '); + }); + } catch (ex) { + com_util.renderAlertModal('Not applicable file contents with vp format! (JSON)'); + } + }).catch(function(err) { + vpLog.display(VP_LOG_TYPE.ERROR, err); + }); + } else { + fetch(selectedPath).then(function(file) { if (file.status != 200) { alert("The file format is not valid."); return; @@ -159,6 +194,8 @@ define([ com_util.renderSuccessMessage(fileName + ' imported '); }); }); + } + }); } @@ -334,7 +371,7 @@ define([ if(selected || that.clicked > 1) { // double click or clicked after selection // enable input - $(thisHeader).find('.vp-sn-item-title').prop('disabled', false); + $(thisHeader).find('.vp-sn-item-title').prop('readonly', false); $(thisHeader).find('.vp-sn-item-title').select(); $(thisHeader).find('.vp-sn-item-title').focus(); @@ -376,7 +413,7 @@ define([ // prevent occuring header click event by clicking input $(this.wrapSelector('.vp-sn-item-title')).off('click'); $(this.wrapSelector('.vp-sn-item-title')).on('click', function(evt) { - evt.stopPropagation(); + // evt.stopPropagation(); }); // item title save @@ -421,7 +458,7 @@ define([ } // disable - $(this).prop('disabled', true); + $(this).prop('readonly', true); }); // item menu click @@ -435,14 +472,16 @@ define([ let cmCode = that.codemirrorList[title]; cmCode.save(); var code = cmCode.getValue(); + // DEPRECATED: no longer save to block as default // create block and run it - $('#vp_wrapper').trigger({ - type: 'create_option_page', - blockType: 'block', - menuId: 'lgExe_code', - menuState: { taskState: { code: code } }, - afterAction: 'run' - }); + // $('#vp_wrapper').trigger({ + // type: 'create_option_page', + // blockType: 'block', + // menuId: 'lgExe_code', + // menuState: { taskState: { code: code } }, + // afterAction: 'run' + // }); + com_interface.insertCell('code', code, true, 'Data Analysis > Snippets'); } else if (menu == 'duplicate') { var dupNo = 1; var timestamp = new Date().getTime(); @@ -473,7 +512,7 @@ define([ } else if (menu == 'delete') { let loadingSpinner = new LoadingSpinner($(that.wrapSelector('.vp-sn-table'))); // remove key - if (vpConfig.extensionType === 'lab') { + if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { vpConfig.getData('').then(function(data) { let dataObj = data; // remove data @@ -486,7 +525,7 @@ define([ // vp-multilang for success message com_util.renderSuccessMessage('Successfully removed!'); }).catch(function(err) { - com_util.renderAlertModal('Failed to remove data...', err); + com_util.renderAlertModal('Failed to remove data...', { content: err }); // load again that.loadUdfList(); }).finally(function() { @@ -504,7 +543,7 @@ define([ // vp-multilang for success message com_util.renderSuccessMessage('Successfully removed!'); }).catch(function(err) { - com_util.renderAlertModal('Failed to remove data...', err); + com_util.renderAlertModal('Failed to remove data...', { content: err }); // load again that.loadUdfList(); }).finally(function() { @@ -582,7 +621,7 @@ define([ item.appendFormatLine('

', 'vp-sn-item', title, timestamp); item.appendFormatLine('

', 'vp-sn-item-header'); item.appendFormatLine('

', 'vp-sn-indicator'); - item.appendFormatLine('', 'vp-sn-item-title', title); + item.appendFormatLine('', 'vp-sn-item-title', title); if (hasImported) { item.appendFormatLine('', 'fa fa-circle vp-sn-imported-item'); } @@ -630,21 +669,23 @@ define([ // load udf list to table 'vp_udfList' let loadingSpinner = new LoadingSpinner($(this.wrapSelector('.vp-sn-table'))); vpConfig.getData().then(function(udfObj) { - vpLog.display(VP_LOG_TYPE.DEVELOP, udfObj); + vpLog.display(VP_LOG_TYPE.DEVELOP, 'Snippets get data', udfObj); var snippets = new com_String(); - Object.keys(udfObj).forEach(key => { - let obj = udfObj[key]; - if (obj.code != null && obj.code != undefined) { - - var hasImported = false; - if (that.importedList.includes(key)) { - // set new label - hasImported = true; + if (udfObj !== undefined) { + Object.keys(udfObj).forEach(key => { + let obj = udfObj[key]; + if (obj.code != null && obj.code != undefined) { + + var hasImported = false; + if (that.importedList.includes(key)) { + // set new label + hasImported = true; + } + var item = that.renderSnippetItem(key, obj.code, obj.timestamp, hasImported); + snippets.append(item); } - var item = that.renderSnippetItem(key, obj.code, obj.timestamp, hasImported); - snippets.append(item); - } - }); + }); + } $(that.wrapSelector('.vp-sn-table')).html(snippets.toString()); // bind snippet item @@ -657,7 +698,8 @@ define([ that.bindCodeMirror(title, tag); }); }).catch(function(err) { - com_util.renderAlertModal(err); + // com_util.renderAlertModal(err); + vpLog.display(VP_LOG_TYPE.ERROR, err); }).finally(function() { loadingSpinner.remove(); }); diff --git a/visualpython/js/m_apps/Subset.js b/visualpython/js/m_apps/Subset.js index 2747ac6c..41af3ef4 100644 --- a/visualpython/js/m_apps/Subset.js +++ b/visualpython/js/m_apps/Subset.js @@ -21,11 +21,25 @@ define([ 'vp_base/js/com/component/PopupComponent', 'vp_base/js/com/component/SuggestInput', 'vp_base/js/com/component/VarSelector', - 'vp_base/js/com/component/MultiSelector' -], function(subsetHtml, subsetCss, com_Const, com_String, com_util, PopupComponent, SuggestInput, VarSelector, MultiSelector) { + 'vp_base/js/com/component/DataSelector' +], function(subsetHtml, subsetCss, com_Const, com_String, com_util, PopupComponent, SuggestInput, VarSelector, DataSelector) { /** * Subset + * ==================================== + * Special mode + * 1. useAsModule : Use subset as module like DataSelector + * - No allocation + * - No run to cell (able to use apply button instead) + * - renders button to target + * 2. useInputVariable : Use subset as module but use applied variable + * - No allocation + * - No data selection + * - No run to cell + * - renders button to target + * 3. useInputColumns : Use subset as module but use applied columns + * - No allocation + * - No column selection */ class Subset extends PopupComponent { _init() { @@ -39,8 +53,9 @@ define([ this.targetSelector = this.prop.targetSelector; this.pageThis = this.prop.pageThis; + this.useAsModule = this.prop.useAsModule; this.useInputVariable = this.prop.useInputVariable; - if (this.useInputVariable) { + if (this.useInputVariable === true || this.useAsModule === true) { this.eventTarget = this.targetSelector; this.useCell = false; // show apply button only } @@ -77,6 +92,7 @@ define([ useCopy: false, toFrame: false, subsetType: 'loc', // subset / loc / iloc / query + returnType: '', rowType: 'condition', rowList: [], @@ -115,6 +131,13 @@ define([ this.loadStateAfterRender(); // render button + if (this.useAsModule) { + // render button + this.renderButton(); + + // hide allocate to + $(this.wrapSelector('.' + VP_DS_ALLOCATE_TO)).closest('tr').hide(); + } if (this.useInputVariable) { // set readonly $(this.wrapSelector('.' + VP_DS_PANDAS_OBJECT)).attr('disabled', true); @@ -142,7 +165,19 @@ define([ } templateForBody() { - return subsetHtml; + let page = $(subsetHtml); + + let that = this; + // Removed dataselector for Allocation input + // let allocateSelector = new DataSelector({ + // pageThis: this, id: 'allocateTo', classes: VP_DS_ALLOCATE_TO, placeholder: 'New variable name', + // finish: function() { + // that.generateCode(); + // } + // }); + // $(page).find('.' + VP_DS_ALLOCATE_TO).replaceWith(allocateSelector.toTagString()); + + return page; } templateForDataView() { let tag = new com_String(); @@ -151,7 +186,7 @@ define([ VP_DS_DATA_VIEW_ALL_DIV, VP_DS_DATA_VIEW_ALL, (this.state.viewAll?'checked':''), "view all"); // data view tag.appendFormatLine('

', VP_DS_DATA_VIEW_BOX, - 'rendered_html'); // 'rendered_html' style from jupyter output area + 'vp_rendered_html'); // 'rendered_html' style from jupyter output area return tag.toString(); } getAllowSubsetTypes() { @@ -164,7 +199,7 @@ define([ buttonTag.appendFormat('', VP_DS_BTN, this.uuid, 'vp-button', 'Subset'); if (this.pageThis) { - $(this.targetSelector).parent().append(buttonTag.toString()); + $(buttonTag.toString()).insertAfter($(this.targetSelector)); } } renderSubsetType(dataType) { @@ -308,12 +343,15 @@ define([ $(this.wrapSelector('.select-row .vp-ds-select-box.left')).on('scroll', function() { if ($(this).scrollTop() + $(this).innerHeight() >= ($(this)[0].scrollHeight - 2)) { let scrollPos = $(this).scrollTop(); + if (that.state.rowLimit > that.state.rowList.length){ + return; // Prevents scroll from being fixed downwards + } let start = that.state.rowLimit; let end = start + 10; let subsetVariable = com_util.formatString('{0}.iloc[{1}:{2}]', that.state.pandasObject, start, end); - vpKernel.getRowList(subsetVariable).then(function (resultObj) { + vpKernel.getRowList(subsetVariable, start).then(function (resultObj) { let { result } = resultObj; - var rowList = JSON.parse(result); + var { list:rowList } = JSON.parse(result); rowList = rowList.map(function (x) { return { ...x, @@ -326,9 +364,9 @@ define([ rowList = rowList.map(function (x) { return { ...x, - label: x.location + '', - value: x.location + '', - code: x.location + '', + label: x.label + '', + value: x.value + '', + code: x.code + '', }; }); } @@ -456,13 +494,13 @@ define([ // tag.appendFormatLine('', // VP_DS_SELECT_DEL_ALL_BTN, 'select-col', 'Remove all items', com_Const.IMAGE_PATH + 'arrow_left_double.svg'); tag.appendFormatLine('', - VP_DS_SELECT_ADD_ALL_BTN, 'select-row', 'Add all items'); + VP_DS_SELECT_ADD_ALL_BTN, 'select-col', 'Add all items'); tag.appendFormatLine('', - VP_DS_SELECT_ADD_BTN, 'select-row'); + VP_DS_SELECT_ADD_BTN, 'select-col'); tag.appendFormatLine('', - VP_DS_SELECT_DEL_BTN, 'select-row'); + VP_DS_SELECT_DEL_BTN, 'select-col'); tag.appendFormatLine('', - VP_DS_SELECT_DEL_ALL_BTN, 'select-row', 'Remove all items'); + VP_DS_SELECT_DEL_ALL_BTN, 'select-col', 'Remove all items'); tag.appendLine('

'); // VP_DS_SELECT_BTNS // col select - right @@ -489,8 +527,14 @@ define([ // col.array parsing var colInfo = com_util.safeString(col.array); // render column box - tag.appendFormatLine('

{8}

', - VP_DS_SELECT_ITEM, 'select-col', VP_DS_DRAGGABLE, col.location, col.value, col.dtype, col.code, col.label + ': \n' + colInfo, col.label); + let numIconStr = ''; + if (col.isNumeric === true) { + numIconStr = ''; + } else { + numIconStr = ''; + } + tag.appendFormatLine('

{8}{9}

', + VP_DS_SELECT_ITEM, 'select-col', VP_DS_DRAGGABLE, col.location, col.value, col.dtype, col.code, col.label + ': \n' + colInfo, numIconStr, col.label); }); tag.appendLine('

'); // VP_DS_SELECT_BOX $(this.wrapSelector('.select-col .' + VP_DS_SELECT_BOX + '.left')).replaceWith(function () { @@ -552,6 +596,7 @@ define([ renderColumnConditionList(colList) { var tag = new com_String(); tag.appendFormatLine('

	{8}
	{6}
{4}	{0}
{4}	{0}

', VP_DS_CONDITION_TBL); + tag.appendLine(this.templateForConditionBox(colList)); tag.appendLine(''); tag.appendFormatLine('', VP_DS_BUTTON_ADD_CONDITION, 'vp-add-col', '+ Condition'); @@ -695,20 +740,20 @@ define([ } } else { var errorContent = ''; - if (msg.content.ename) { + if (msg.content && msg.content.ename) { errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue); } that.renderDataPage(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); } }).catch(function(resultObj) { let { msg } = resultObj; var errorContent = ''; - if (msg.content.ename) { + if (msg.content && msg.content.ename) { errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue); } that.renderDataPage(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); }); } /** @@ -732,6 +777,7 @@ define([ var varType = JSON.parse(result); that.state.pandasObject = prevValue; that.state.dataType = varType; + that.state.returnType = varType; $(that.wrapSelector('.' + VP_DS_PANDAS_OBJECT_BOX)).replaceWith(function () { return $(com_util.formatString('

', 'vp-input', VP_DS_PANDAS_OBJECT, prevValue)); @@ -772,6 +818,7 @@ define([ $(this.wrapSelector()).data('dtype', item.dtype); that.state.pandasObject = value; that.state.dataType = item.dtype; + that.state.returnType = item.dtype; $(this.wrapSelector()).trigger('change'); }); variableInput.setNormalFilter(true); @@ -1134,13 +1181,6 @@ define([ that.focus(); }); $(this.targetSelector).on('apply_option_page', function(evt) { - let code = that.generateCode(); - - // if finish callback is available - if (that.finish && typeof that.finish == 'function') { - that.finish(code); - } - that.close(); }); } @@ -1157,6 +1197,7 @@ define([ that.state.pandasObject = varName; that.state.dataType = event.dataType ? event.dataType : that.state.dataType; + that.state.returnType = that.state.dataType; that.state.rowList = []; that.state.rowLimit = 10; that.state.columnList = []; @@ -1175,15 +1216,16 @@ define([ // get result and load column list vpKernel.getColumnList(varName).then(function (resultObj) { let { result } = resultObj; - var colList = JSON.parse(result); - colList = colList.map(function (x) { + var { list } = JSON.parse(result); + list = list.map(function (x) { return { ...x, value: x.label, code: x.value, + isNumeric: x.is_numeric }; }); - that.loadColumnList(colList); + that.loadColumnList(list); that.bindDraggable('col'); that.generateCode(); }); @@ -1192,7 +1234,7 @@ define([ let subsetVariable = com_util.formatString('{0}.iloc[:{1}]', varName, that.state.rowLimit); vpKernel.getRowList(subsetVariable).then(function (resultObj) { let { result } = resultObj; - var rowList = JSON.parse(result); + var { list:rowList } = JSON.parse(result); rowList = rowList.map(function (x) { return { ...x, @@ -1225,7 +1267,7 @@ define([ let subsetVariable = com_util.formatString('{0}.iloc[:{1}]', varName, that.state.rowLimit); vpKernel.getRowList(subsetVariable).then(function (resultObj) { let { result } = resultObj; - var rowList = JSON.parse(result); + var { list:rowList } = JSON.parse(result); rowList = rowList.map(function (x) { return { ...x, @@ -1703,7 +1745,7 @@ define([ var connector = i > 0 ? $(condList[i - 1]).find('.vp-oper-connect').val() : undefined; // if no variable selected, pass - if (varName == "") + if (varName === "" || oper === "") continue; if (useCondition) { rowSelection.append(connector); @@ -1771,6 +1813,10 @@ define([ rowSelection.append(')'); } else { rowSelection.appendFormat('({0}', varName); + if (colName == '.index') { + // index + rowSelection.append('.index'); + } oper && rowSelection.appendFormat(' {0}', oper); if (cond) { // condition value as text @@ -1809,10 +1855,14 @@ define([ if (this.state.colType == 'indexing') { if (this.useInputColumns == true) { colList = this.state.selectedColumns; - if (colList.length == 1) { - colSelection.appendFormat('{0}', colList.toString()); - } else { - colSelection.appendFormat('[{0}]', colList.toString()); + if (colList.length > 0) { + if (colList.length == 1) { + colSelection.appendFormat('{0}', colList.toString()); + this.state.returnType = 'Series'; + } else { + colSelection.appendFormat('[{0}]', colList.toString()); + this.state.returnType = 'DataFrame'; + } } } else { var colTags = $(this.wrapSelector('.' + VP_DS_SELECT_ITEM + '.select-col.added:not(.moving)')); @@ -1820,7 +1870,7 @@ define([ var colList = []; for (var i = 0; i < colTags.length; i++) { var colValue = $(colTags[i]).data('code'); - if (colValue) { + if (colValue !== undefined) { colList.push(colValue); } } @@ -1830,13 +1880,16 @@ define([ $(this.wrapSelector('.' + VP_DS_TO_FRAME)).parent().show(); // to frame - if (this.state.toFrame) { + if (this.state.toFrame === true) { colSelection.appendFormat('[{0}]', colList.toString()); + this.state.returnType = 'DataFrame'; } else { colSelection.appendFormat('{0}', colList.toString()); + this.state.returnType = 'Series'; } } else { colSelection.appendFormat('[{0}]', colList.toString()); + this.state.returnType = 'DataFrame'; } } else { @@ -1851,31 +1904,34 @@ define([ } // use simple selection - if (this.state.subsetType == 'subset') { - if (rowSelection.toString() != ':' && rowSelection.toString() != '') { - code.appendFormat('[{0}]', rowSelection.toString()); - } - if (colSelection.toString() != ':' && colSelection.toString() != '') { - code.appendFormat('[{0}]', colSelection.toString()); - } - } else if (this.state.subsetType == 'loc') { - if (this.state.dataType == 'DataFrame') { - code.appendFormat('.loc[{0}, {1}]', rowSelection.toString(), colSelection.toString()); - } else { - code.appendFormat('.loc[{0}]', rowSelection.toString()); - } - } else if (this.state.subsetType == 'iloc') { - if (this.state.dataType == 'DataFrame') { - code.appendFormat('.iloc[{0}, {1}]', rowSelection.toString(), colSelection.toString()); - } else { - code.appendFormat('.iloc[{0}]', rowSelection.toString()); - } - } else if (this.state.subsetType == 'query') { - if (rowSelection.toString() != ':' && rowSelection.toString() != '') { - code.appendFormat('.query("{0}")', rowSelection.toString()); - } - if (colSelection.toString() != ':' && colSelection.toString() != '') { - code.appendFormat('[{0}]', colSelection.toString()); + if ((rowSelection.toString() !== ':' && rowSelection.toString() !== '') + || (colSelection.toString() !== ':' && colSelection.toString() !== '')) { + if (this.state.subsetType == 'subset') { + if (rowSelection.toString() != ':' && rowSelection.toString() != '') { + code.appendFormat('[{0}]', rowSelection.toString()); + } + if (colSelection.toString() != ':' && colSelection.toString() != '') { + code.appendFormat('[{0}]', colSelection.toString()); + } + } else if (this.state.subsetType == 'loc') { + if (this.state.dataType == 'DataFrame') { + code.appendFormat('.loc[{0}, {1}]', rowSelection.toString(), colSelection.toString()); + } else { + code.appendFormat('.loc[{0}]', rowSelection.toString()); + } + } else if (this.state.subsetType == 'iloc') { + if (this.state.dataType == 'DataFrame') { + code.appendFormat('.iloc[{0}, {1}]', rowSelection.toString(), colSelection.toString()); + } else { + code.appendFormat('.iloc[{0}]', rowSelection.toString()); + } + } else if (this.state.subsetType == 'query') { + if (rowSelection.toString() != ':' && rowSelection.toString() != '') { + code.appendFormat('.query("{0}")', rowSelection.toString()); + } + if (colSelection.toString() != ':' && colSelection.toString() != '') { + code.appendFormat('[{0}]', colSelection.toString()); + } } } @@ -1907,9 +1963,11 @@ define([ if (this.useInputVariable) { this.loadVariables(); this.reloadSubsetData(); + } + if (this.useCell === false) { // show save button only this.setSaveOnlyMode(); - } + } // generate code after displaying page // - codemirror can be set after display this.generateCode(); @@ -1920,13 +1978,13 @@ define([ //==================================================================== hideButton() { - if (this.useInputVariable) { + if (this.useInputVariable === true || this.useAsModule === true) { $(this.pageThis.wrapSelector('.' + VP_DS_BTN + '.' + this.uuid)).hide(); } } disableButton() { - if (this.useInputVariable) { + if (this.useInputVariable === true || this.useAsModule === true) { var buttonEle = $(this.pageThis.wrapSelector('.' + VP_DS_BTN + '.' + this.uuid)); if (!buttonEle.hasClass('disabled')) { buttonEle.addClass('disabled'); @@ -1935,12 +1993,12 @@ define([ } enableButton() { - if (this.useInputVariable) { + if (this.useInputVariable === true || this.useAsModule === true) { $(this.pageThis.wrapSelector('.' + VP_DS_BTN + '.' + this.uuid)).removeClass('disabled'); } } showButton() { - if (this.useInputVariable) { + if (this.useInputVariable === true || this.useAsModule === true) { $(this.pageThis.wrapSelector('.' + VP_DS_BTN + '.' + this.uuid)).show(); } } diff --git a/visualpython/js/m_apps/Sweetviz.js b/visualpython/js/m_apps/Sweetviz.js new file mode 100644 index 00000000..2059189c --- /dev/null +++ b/visualpython/js/m_apps/Sweetviz.js @@ -0,0 +1,370 @@ +/* + * Project Name : Visual Python + * Description : GUI-based Python code generator + * File Name : Sweetviz.js + * Author : Black Logic + * Note : Apps > Sweetviz + * License : GNU GPLv3 with Visual Python special exception + * Date : 2021. 11. 18 + * Change Date : + */ + +//============================================================================ +// [CLASS] Sweetviz +//============================================================================ +define([ + __VP_TEXT_LOADER__('vp_base/html/m_apps/sweetviz.html'), // INTEGRATION: unified version of text loader + __VP_CSS_LOADER__('vp_base/css/m_apps/sweetviz'), // INTEGRATION: unified version of css loader + 'vp_base/js/com/com_Const', + 'vp_base/js/com/com_String', + 'vp_base/js/com/com_interface', + 'vp_base/js/com/component/PopupComponent', + 'vp_base/js/com/component/SuggestInput', + 'vp_base/js/com/component/FileNavigation' +], function(svHtml, svCss, com_Const, com_String, com_interface, PopupComponent, SuggestInput, FileNavigation) { + + const PROFILE_TYPE = { + NONE: -1, + GENERATE: 1 + } + + const LIST_MENU_ITEM = { + SHOW: 'show', + DELETE: 'delete', + SAVE: 'save' + } + + /** + * Sweetviz + */ + class Sweetviz extends PopupComponent { + _init() { + super._init(); + /** Write codes executed before rendering */ + this.config.installButton = true; + this.config.importButton = true; + this.config.codeview = false; + this.config.dataview = false; + this.config.runButton = false; + this.config.size = { width: 500, height: 500 }; + this.config.checkModules = ['sweetviz']; + + this.selectedReport = ''; + } + + _unbindEvent() { + super._unbindEvent(); + $(this.wrapSelector('.vp-sv-df-refresh')).off('click'); + $(this.wrapSelector('#vp_pfPathButton')).off('click'); + $(this.wrapSelector('.vp-sv-menu-item')).off('click'); + } + + _bindEvent() { + super._bindEvent(); + /** Implement binding events */ + let that = this; + + // refresh df + $(this.wrapSelector('.vp-sv-df-refresh')).on('click', function() { + that.loadVariableList(); + }); + + // click path + $(this.wrapSelector('#vp_pfPathButton')).on('click', function() { + let fileNavi = new FileNavigation({ + type: 'save', + extensions: ['html'], + finish: function(filesPath, status, error) { + let {file, path} = filesPath[0]; + + // set text + $(that.wrapSelector('#vp_pfPath')).data('file', file); + $(that.wrapSelector('#vp_pfPath')).val(path); + } + }); + fileNavi.open(); + }); + + // click menu + $(this.wrapSelector('.vp-sv-menu-item')).on('click', function() { + // check required filled + if (that.checkRequiredOption() === false) { + return ; + } + + var type = $(this).data('type'); + var df = $(that.wrapSelector('#vp_pfVariable')).val(); + var saveas = $(that.wrapSelector('#vp_pfReturn')).val(); + if (saveas == '') { + saveas = '_vp_sweetviz'; + } + var title = $(that.wrapSelector('#vp_pfTitle')).val(); + var filePath = $(that.wrapSelector('#vp_pfPath')).val(); + // var openBrowser = $(that.wrapSelector('#vp_pfOpenBrowser')).prop('checked'); + var code = new com_String(); + switch(parseInt(type)) { + case PROFILE_TYPE.GENERATE: + // generate with/out title + if (title && title != '') { + code.appendFormatLine("{0} = sweetviz.analyze([{1}, '{2}'])", saveas, df, title); + } else { + code.appendFormatLine("{0} = sweetviz.analyze({1})", saveas, df); + } + // show notebook + code.appendFormat("{0}.show_notebook(", saveas); + if (filePath && filePath != '') { + code.appendFormat("filepath='{0}'", filePath); + } + // if (openBrowser === false) { + // code.append(", open_browser=False"); + // } + code.appendLine(')'); + code.append(saveas); + break; + } + that.checkAndRunModules(true).then(function() { + com_interface.insertCell('code', code.toString(), true, 'Data Analysis > Sweetviz'); + that.loadReportList(); + }); + }); + } + + bindReportListEvent() { + let that = this; + // click list item menu + $(this.wrapSelector('.vp-sv-list-menu-item')).off('click'); + $(this.wrapSelector('.vp-sv-list-menu-item')).on('click', function(evt) { + var menu = $(this).data('menu'); + var itemTag = $(this).closest('.vp-sv-list-item'); + var varName = $(itemTag).data('name'); + + var code = new com_String(); + switch(menu) { + case LIST_MENU_ITEM.SHOW: + code.appendFormat("{0}.show_notebook()", varName); + break; + case LIST_MENU_ITEM.DELETE: + code.appendFormat("del {0}", varName); + break; + case LIST_MENU_ITEM.SAVE: + let fileNavi = new FileNavigation({ + type: 'save', + extensions: ['html'], + fileName: 'report', + finish: function(filesPath, status, error) { + filesPath.forEach( fileObj => { + var fileName = fileObj.file; + var path = fileObj.path; + if (varName == '') { + varName = '_vp_sweetviz'; + } + var code = new com_String(); + code.appendFormat("{0}.show_html(filepath='{1}')", varName, path); + com_interface.insertCell('code', code.toString(), true, 'Data Analysis > Sweetviz'); + + that.selectedReport = ''; + }); + } + }); + fileNavi.open(); + return; + default: + return; + } + com_interface.insertCell('code', code.toString(), true, 'Data Analysis > Sweetviz'); + that.loadReportList(); + }); + } + + templateForBody() { + return svHtml; + } + + render() { + super.render(); + + this.loadVariableList(); + this.loadReportList(); + this.checkInstalled(); + } + + generateInstallCode() { + return [ + '!pip install sweetviz' + ]; + } + + generateImportCode() { + return [ + 'import sweetviz' + ]; + } + + generateCode() { + return ""; + } + + loadVariableList() { + var that = this; + // load using kernel + var dataTypes = ['DataFrame']; + vpKernel.getDataList(dataTypes).then(function(resultObj) { + try { + let { result, msg } = resultObj; + var varList = JSON.parse(result); + // render variable list + // replace + $(that.wrapSelector('#vp_pfVariable')).replaceWith(function() { + return that.templateForVariableList(varList); + }); + $(that.wrapSelector('#vp_pfVariable')).trigger('change'); + } catch (ex) { + vpLog.display(VP_LOG_TYPE.ERROR, 'Sweetviz:', result); + } + }); + } + + templateForVariableList(varList) { + var beforeValue = $(this.wrapSelector('#vp_pfVariable')).val(); + if (beforeValue == null) { + beforeValue = ''; + } + + let mappedList = varList.map(obj => { return { label: obj.varName, value: obj.varName, dtype: obj.varType } }); + + var variableInput = new SuggestInput(); + variableInput.setComponentID('vp_pfVariable'); + variableInput.addClass('vp-sv-select'); + variableInput.setPlaceholder('Select variable'); + variableInput.setSuggestList(function () { return mappedList; }); + variableInput.setNormalFilter(true); + variableInput.addAttribute('required', true); + variableInput.setValue(beforeValue); + + return variableInput.toTagString(); + } + + /** + * Toggle check button state + * @param {String} mode install/checking/installed + */ + toggleCheckState(mode='checking') { + let message = 'Checking...'; + let disable = true; + switch (mode) { + case 'install': + message = 'Install'; + disable = false; + break; + case 'installed': + message = 'Installed'; + disable = true; + break; + case 'installing': + message = 'Installing'; + disable = true; + break; + } + $(this.wrapSelector('.vp-sv-install-btn')).text(message); + if (disable) { + // set state as 'Checking' + // set disabled + if (!$(this.wrapSelector('.vp-sv-install-btn')).hasClass('disabled')) { + $(this.wrapSelector('.vp-sv-install-btn')).addClass('disabled'); + } + } else { + // set enabled + $(this.wrapSelector('.vp-sv-install-btn')).removeClass('disabled'); + } + } + + checkInstalled() { + var that = this; + // set state as 'Checking' + this.toggleCheckState(); + this.checking = true; + + // check installed + let code = "_vp_print(_vp_check_package_list(['sweetviz']))"; + vpKernel.execute(code).then(function(resultObj) { + let { result, msg } = resultObj; + if (!that.checking) { + return; + } + let installed = result['sweetviz'].installed; + if (installed === false) { + that.toggleCheckState('install'); + } else { + that.toggleCheckState('installed'); + } + that.checking = false; + }).catch(function(err) { + that.toggleCheckState('install'); + that.checking = false; + }); + } + + loadReportList() { + var that = this; + // load using kernel + vpKernel.getSweetvizList().then(function(resultObj) { + try { + let { result } = resultObj; + var varList = JSON.parse(result); + // render variable list + // replace + $(that.wrapSelector('.vp-sv-list-box')).replaceWith(function() { + return that.renderReportList(varList); + }); + + that.bindReportListEvent(); + } catch (ex) { + vpLog.display(VP_LOG_TYPE.ERROR, 'Sweetviz:', result); + // console.log(ex); + } + }); + } + + renderReportList = function(reportList=[]) { + var page = new com_String(); + page.appendFormatLine('

', 'vp-sv-list-box'); + page.appendFormatLine('

', 'vp-sv-list-header'); + page.appendFormatLine('

{1}

', 'vp-sv-list-header-item', 'Allocated to'); + page.appendFormatLine('

{1}

', 'vp-sv-list-header-item', 'Report Title'); + page.appendFormatLine('

{1}

', 'vp-sv-list-header-item', ''); + page.appendLine('

'); + page.appendFormatLine('

', 'vp-apiblock-scrollbar'); + page.appendFormatLine('

', 'vp-sv-list-body', 'vp-apiblock-scrollbar'); + reportList.forEach((report, idx) => { + var { varName, title } = report; + page.appendFormatLine('

', 'vp-sv-list-item', varName, title); + page.appendFormatLine('

{0}

', varName); + page.appendFormatLine('

{0}

', title); + // button box + page.appendFormatLine('

', 'vp-sv-list-button-box'); + // LAB: img to url + // page.appendFormatLine('

' + // , 'vp-sv-list-menu-item', LIST_MENU_ITEM.SHOW, 'Show report', com_Const.IMAGE_PATH + 'snippets/run.svg'); + // page.appendFormatLine('

' + // , 'vp-sv-list-menu-item', LIST_MENU_ITEM.DELETE, 'Delete report', com_Const.IMAGE_PATH + 'delete.svg'); + // page.appendFormatLine('

' + // , 'vp-sv-list-menu-item', LIST_MENU_ITEM.SAVE, 'Save report', com_Const.IMAGE_PATH + 'snippets/export.svg'); + page.appendFormatLine('

' + , 'vp-sv-list-menu-item', 'vp-icon-run', LIST_MENU_ITEM.SHOW, 'Show report'); + page.appendFormatLine('

' + , 'vp-sv-list-menu-item', 'vp-icon-delete', LIST_MENU_ITEM.DELETE, 'Delete report'); + page.appendFormatLine('

' + , 'vp-sv-list-menu-item', 'vp-icon-export', LIST_MENU_ITEM.SAVE, 'Save report'); + page.appendLine('

'); + page.appendLine('

'); + }); + page.appendLine('

'); // VP_PF_LIST_BODY + page.appendLine('

'); + page.appendLine('

'); // 'vp-sv-list-box' + return page.toString(); + } + + } + + return Sweetviz; +}); \ No newline at end of file diff --git a/visualpython/js/m_library/m_pandas/getValueCounts.js b/visualpython/js/m_library/m_pandas/getValueCounts.js index 837e2033..3fe3c44f 100644 --- a/visualpython/js/m_library/m_pandas/getValueCounts.js +++ b/visualpython/js/m_library/m_pandas/getValueCounts.js @@ -33,12 +33,12 @@ define([ super.render(); // add var selector - var varSelector = new VarSelector(['DataFrame', 'Series', 'Index'], 'DataFrame', false); - varSelector.setComponentId('i0'); - varSelector.addClass('vp-state'); - varSelector.setUseColumn(true); - varSelector.setValue(this.state.i0); - $(this.wrapSelector('#i0')).replaceWith(varSelector.render()); + // var varSelector = new VarSelector(['DataFrame', 'Series', 'Index'], 'DataFrame', false); + // varSelector.setComponentId('i0'); + // varSelector.addClass('vp-state'); + // varSelector.setUseColumn(true); + // varSelector.setValue(this.state.i0); + // $(this.wrapSelector('#i0')).replaceWith(varSelector.render()); } } diff --git a/visualpython/js/m_library/m_pandas/readFile.js b/visualpython/js/m_library/m_pandas/readFile.js index 55b133ec..c9144db9 100644 --- a/visualpython/js/m_library/m_pandas/readFile.js +++ b/visualpython/js/m_library/m_pandas/readFile.js @@ -19,7 +19,7 @@ define([ 'vp_base/js/com/com_util', 'vp_base/js/com/com_Const', 'vp_base/js/com/component/PopupComponent', - 'vp_base/js/com/com_generator', + 'vp_base/js/com/com_generatorV2', 'vp_base/data/m_library/pandasLibrary', 'vp_base/js/com/component/FileNavigation', 'vp_base/js/com/component/SuggestInput' @@ -48,20 +48,23 @@ define([ 'csv': 'csv', 'excel': 'xlsx', 'json': 'json', - 'pickle': 'pickle' + 'pickle': '', + 'parquet': 'parquet' } - this.dataPath = window.location.origin + com_Const.DATA_PATH + "sample_csv/"; + this.dataPath = 'https://raw.githubusercontent.com/visualpython/visualpython/main/visualpython/data/sample_csv/'; this.fileResultState = { pathInputId : this.wrapSelector('#i0'), fileInputId : this.wrapSelector('#fileName') }; this.fileState = { fileTypeId: { - 'csv': 'pd004', - 'excel': 'pd123', - 'json': 'pd076', - 'pickle': 'pd079' + 'csv': 'pd_readCsv', + 'excel': 'pd_readExcel', + 'json': 'pd_readJson', + 'pickle': 'pd_readPickle', + 'parquet': 'pd_readParquet' }, + selectedType: 'csv', package: null }; } @@ -145,6 +148,7 @@ define([

@@ -153,7 +157,7 @@ define([

- +

@@ -153,7 +157,7 @@ define([

- +

@@ -175,8 +179,8 @@ define([ var that = this; // clear - $(this.wrapSelector('#vp_inputOutputBox table')).html(''); - $(this.wrapSelector('#vp_optionBox table')).html(''); + $(this.wrapSelector('#vp_inputOutputBox table')).html(''); + $(this.wrapSelector('#vp_optionBox table')).html(''); var fileTypeObj = this.fileState['fileTypeId']; var selectedType = this.state['selectedType']; @@ -196,10 +200,11 @@ define([ } // render interface - pdGen.vp_showInterfaceOnPage(this.wrapSelector('.vp-fileio-box'), thisPkg); + // pdGen.vp_showInterfaceOnPage(this.wrapSelector('.vp-fileio-box'), thisPkg); + pdGen.vp_showInterfaceOnPage(this, thisPkg, this.state); // prepend file type selector - $(this.wrapSelector('#vp_inputOutputBox table')).prepend( + $(this.wrapSelector('#vp_inputOutputBox table tbody')).prepend( $('').append($(`File Type`)) .append($('')) ); @@ -250,11 +255,12 @@ define([ this.saveState(); var thisPkg = JSON.parse(JSON.stringify(this.fileState.package)); - thisPkg.input.push({ + thisPkg.options.push({ name: 'fileType', type: 'var' }); - var result = pdGen.vp_codeGenerator(this.uuid, thisPkg); + // var result = pdGen.vp_codeGenerator(this.uuid, thisPkg); + var result = pdGen.vp_codeGenerator(this, thisPkg, this.state); sbCode.append(result); return sbCode.toString(); diff --git a/visualpython/js/m_logic/Comment.js b/visualpython/js/m_logic/Comment.js index 83ba3e75..fdec26fe 100644 --- a/visualpython/js/m_logic/Comment.js +++ b/visualpython/js/m_logic/Comment.js @@ -18,6 +18,51 @@ define([ ], function(com_String, PopupComponent) { const COMMENT_DEFAULT_CODE = '# Write down comments' + // Templates from NumPy Style Python Docstrings. + const COMMENT_CLASS_TEMPLATE = +`"""Summarize the class in one line. +Several sentences ... +providing an extended description. +Note +---------- +Note about this class. +Parameters +---------- +param1 : param_type + Parameter description. +param2 : param_type + Parameter description. +Attributes +---------- +attr1 : attr_type + Attibute description. +attr2 : attr_type + Attibute description. +Examples +---------- + +References +---------- +"""` + const COMMENT_METHOD_TEMPLATE = +`"""Summarize the function in one line. +Several sentences ... +providing an extended description. +Parameters +---------- +param1 : param_type + Parameter description. +param2 : param_type + Parameter description. +Returns +------- +return_type + Return description. +Note +---------- +Examples +---------- +"""` /** * Comment @@ -29,9 +74,13 @@ define([ this.config.dataview = false; this.config.codeview = false; this.config.saveOnly = true; + this.selectBoxClassName = 'vp-ct-option'; // Select Box ClassName + this.selectOptionPrefix = 'vp_template_'; // Select Options Class Prefix Name this.state = { - code: COMMENT_DEFAULT_CODE, + v1: {"name": "Default", "code": COMMENT_DEFAULT_CODE}, + v2 : {"name": "Class", "code": COMMENT_CLASS_TEMPLATE}, + v3 : {"name": "Method", "code": COMMENT_METHOD_TEMPLATE}, ...this.state } @@ -40,14 +89,45 @@ define([ _bindEvent() { super._bindEvent(); - /** Implement binding events */ + + var commentTemplates = this.state; + let cmCodeListTemp = this.cmCodeList; + + $('.' + this.selectBoxClassName).on('change', function(){ + // get code mirror object + let targetCmObj = cmCodeListTemp.filter(obj => obj.key == 'code'); + let cm = targetCmObj[0].cm; + var templateOption = $(this).val(); + + // Change Code Mirror Text + if(templateOption == commentTemplates.v1['name']){ + cm.setValue(commentTemplates.v1['code']); + }else if(templateOption == commentTemplates.v2['name']){ + cm.setValue(commentTemplates.v2['code']); + }else if(templateOption == commentTemplates.v3['name']){ + cm.setValue(commentTemplates.v3['code']); + } + cm.save(); + }); } templateForBody() { /** Implement generating template */ var page = new com_String(); page.appendFormatLine('

{0}

' - , this.state.code); + , this.state.v1['code']); + + // add select box + page.appendFormatLine(''); + return page.toString(); } diff --git a/visualpython/js/m_logic/Def.js b/visualpython/js/m_logic/Def.js index c5062925..0dd2e11e 100644 --- a/visualpython/js/m_logic/Def.js +++ b/visualpython/js/m_logic/Def.js @@ -77,6 +77,7 @@ define([ } loadState() { + super.loadState(); let { v1, v2 } = this.state; } diff --git a/visualpython/js/m_ml/AutoML.js b/visualpython/js/m_ml/AutoML.js index 18251639..73e30ce0 100644 --- a/visualpython/js/m_ml/AutoML.js +++ b/visualpython/js/m_ml/AutoML.js @@ -88,7 +88,11 @@ define([ let config = that.modelConfig[that.state.modelType]; if (config && config.install != undefined) { // insert install code - com_interface.insertCell('code', config.install, true, 'Machine Learning > AutoML'); + let installCode = config.install; + if (vpConfig.extensionType === 'lite') { + installCode = installCode.replace('!', '%'); + } + com_interface.insertCell('code', installCode, true, 'Machine Learning > AutoML'); } }); diff --git a/visualpython/js/m_ml/Classification.js b/visualpython/js/m_ml/Classification.js index fb4940fe..f327b611 100644 --- a/visualpython/js/m_ml/Classification.js +++ b/visualpython/js/m_ml/Classification.js @@ -50,7 +50,7 @@ define([ this.modelConfig = ML_LIBRARIES; this.modelTypeList = { - 'Classfication': ['lg-rgs', 'bern-nb', 'mulnom-nb', 'gaus-nb', 'sv-clf', 'dt-clf', 'rf-clf', 'gbm-clf', 'xgb-clf', 'lgbm-clf', 'cb-clf'], + 'Classification': ['lg-rgs', 'bern-nb', 'mulnom-nb', 'gaus-nb', 'sv-clf', 'dt-clf', 'rf-clf', 'gbm-clf', 'xgb-clf', 'lgbm-clf', 'cb-clf'], } @@ -88,7 +88,11 @@ define([ let config = that.modelConfig[that.state.modelType]; if (config && config.install != undefined) { // insert install code - com_interface.insertCell('code', config.install, true, 'Machine Learning > Classification'); + let installCode = config.install; + if (vpConfig.extensionType === 'lite') { + installCode = installCode.replace('!', '%'); + } + com_interface.insertCell('code', installCode, true, 'Machine Learning > Classification'); } }); diff --git a/visualpython/js/m_ml/Clustering.js b/visualpython/js/m_ml/Clustering.js index df060086..95ee735c 100644 --- a/visualpython/js/m_ml/Clustering.js +++ b/visualpython/js/m_ml/Clustering.js @@ -51,7 +51,7 @@ define([ this.modelTypeList = { // 'Regression': ['ln-rgs', 'sv-rgs', 'dt-rgs', 'rf-rgs', 'gbm-rgs', 'xgb-rgs', 'lgbm-rgs', 'cb-rgs'], - // 'Classfication': ['lg-rgs', 'sv-clf', 'dt-clf', 'rf-clf', 'gbm-clf', 'xgb-clf', 'lgbm-clf', 'cb-clf'], + // 'Classification': ['lg-rgs', 'sv-clf', 'dt-clf', 'rf-clf', 'gbm-clf', 'xgb-clf', 'lgbm-clf', 'cb-clf'], // 'Auto ML': ['tpot-rgs', 'tpot-clf'], 'Clustering': ['k-means', 'agg-cls', 'gaus-mix', 'dbscan'], // 'Dimension Reduction': ['pca', 'lda', 'svd', 'nmf'] @@ -91,7 +91,11 @@ define([ let config = that.modelConfig[that.state.modelType]; if (config && config.install != undefined) { // insert install code - com_interface.insertCell('code', config.install, true, 'Machine Learning > Clustering'); + let installCode = config.install; + if (vpConfig.extensionType === 'lite') { + installCode = installCode.replace('!', '%'); + } + com_interface.insertCell('code', installCode, true, 'Machine Learning > Clustering'); } }); diff --git a/visualpython/js/m_ml/DataPrep.js b/visualpython/js/m_ml/DataPrep.js index 6683f559..10b9625f 100644 --- a/visualpython/js/m_ml/DataPrep.js +++ b/visualpython/js/m_ml/DataPrep.js @@ -57,13 +57,13 @@ define([ this.modelConfig = ML_LIBRARIES; this.modelTypeList = { - 'Encoding': ['prep-onehot', 'prep-label', 'prep-ordinal', 'prep-target', 'prep-smote'], + 'Encoding': ['prep-onehot', 'prep-label', 'prep-ordinal', 'prep-target'], 'Scaling': ['prep-standard', 'prep-robust', 'prep-minmax', 'prep-normalizer', 'prep-func-trsfrm-log', 'prep-func-trsfrm-exp', 'prep-poly-feat', 'prep-kbins-discretizer'], - 'ETC': ['make-column-transformer'] + 'ETC': ['prep-simple-imputer', 'prep-smote', 'make-column-transformer'] } this.mctEstimator = { - 'Encoding': ['prep-onehot', 'prep-label', 'prep-ordinal', 'prep-target', 'prep-smote'], + 'Encoding': ['prep-onehot', 'prep-label', 'prep-ordinal', 'prep-target'], 'Scaling': ['prep-standard', 'prep-robust', 'prep-minmax', 'prep-normalizer', 'prep-func-trsfrm-log', 'prep-func-trsfrm-exp', 'prep-poly-feat', 'prep-kbins-discretizer'], } @@ -94,10 +94,19 @@ define([ $(that.wrapSelector('#vp_installLibrary')).hide(); } - if (modelType == 'make-column-transformer') { + if (modelType === 'make-column-transformer') { // load mct-targetData that.loadVariableList(); that.bindMCT(); + } else if (modelType === 'prep-simple-imputer') { + $(that.wrapSelector('#missing_values')).replaceWith(`

+ ${$(that.wrapSelector('#missing_values'))[0].outerHTML} + Text +

`); + $(that.wrapSelector('#fill_value')).replaceWith(`

+ ${$(that.wrapSelector('#fill_value'))[0].outerHTML} + Text +

`); } }); @@ -106,7 +115,11 @@ define([ let config = that.modelConfig[that.state.modelType]; if (config && config.install != undefined) { // insert install code - com_interface.insertCell('code', config.install, true, 'Machine Learning > DataPrep'); + let installCode = config.install; + if (vpConfig.extensionType === 'lite') { + installCode = installCode.replace('!', '%'); + } + com_interface.insertCell('code', installCode, true, 'Machine Learning > DataPrep'); } }); @@ -407,6 +420,7 @@ define([ generateCode() { let { modelControlType, modelType, userOption, allocateToCreation, model } = this.state; + let state = JSON.parse(JSON.stringify(this.state)); let code = new com_String(); if (modelControlType == 'creation') { /** @@ -418,22 +432,33 @@ define([ let config = this.modelConfig[modelType]; code.appendLine(config.import); + if (modelType === 'prep-simple-imputer') { + let checkList = ['missing_values', 'fill_value']; + checkList.forEach(checkKey => { + try { + state[checkKey] = com_util.convertToStr(state[checkKey], state[checkKey + '_istext']); + } catch(e) { + ; + } + }); + } + // model code let modelCode = config.code; - modelCode = com_generator.vp_codeGenerator(this, config, this.state, (userOption != ''? ', ' + userOption : '')); + modelCode = com_generator.vp_codeGenerator(this, config, state, (userOption != ''? ', ' + userOption : '')); // generate mct code - if (modelType == 'make-column-transformer') { + if (modelType === 'make-column-transformer') { let mctCodes = []; - let { mct_estimator1, mct_columns1, mct_estimator2, mct_columns2 } = this.state; + let { mct_estimator1, mct_columns1='', mct_estimator2, mct_columns2='' } = state; if (mct_estimator1 != undefined && mct_estimator1 != '') { code.appendLine(this.modelConfig[mct_estimator1].import); - let estimator1code = com_generator.vp_codeGenerator(this, this.modelConfig[mct_estimator1], this.state, (userOption != ''? ', ' + userOption : '')); + let estimator1code = com_generator.vp_codeGenerator(this, this.modelConfig[mct_estimator1], state, (userOption != ''? ', ' + userOption : '')); mctCodes.push(com_util.formatString('({0}, [{1}])', estimator1code, mct_columns1)); } if (mct_estimator2 != undefined && mct_estimator2 != '') { code.appendLine(this.modelConfig[mct_estimator2].import); - let estimator2code = com_generator.vp_codeGenerator(this, this.modelConfig[mct_estimator2], this.state, (userOption != ''? ', ' + userOption : '')); + let estimator2code = com_generator.vp_codeGenerator(this, this.modelConfig[mct_estimator2], state, (userOption != ''? ', ' + userOption : '')); mctCodes.push(com_util.formatString('({0}, [{1}])', estimator2code, mct_columns2)); } modelCode = modelCode.replace('${mct_code}', mctCodes.join(', ')); diff --git a/visualpython/js/m_ml/DataSets.js b/visualpython/js/m_ml/DataSets.js index 6805ff0d..5f0bec3b 100644 --- a/visualpython/js/m_ml/DataSets.js +++ b/visualpython/js/m_ml/DataSets.js @@ -33,7 +33,7 @@ define([ this.config.checkModules = ['pd']; this.state = { - loadType: 'load_boston', + loadType: 'load_iris', userOption: '', allocateTo: 'ldata', ...this.state @@ -42,7 +42,8 @@ define([ this.mlConfig = ML_LIBRARIES; this.loadTypeList = { 'Load Data': [ - 'load_boston', 'load_iris', 'load_diabetes', 'load_digits', 'load_linnerud', 'load_wine', 'load_breast_cancer' + // 'load_boston', // `load_boston` has been removed from scikit-learn since version 1.2. + 'load_iris', 'load_diabetes', 'load_digits', 'load_linnerud', 'load_wine', 'load_breast_cancer' ], 'Create Data': [ 'make_classification', 'make_blobs', 'make_circles', 'make_moons' diff --git a/visualpython/js/m_ml/DimensionReduction.js b/visualpython/js/m_ml/DimensionReduction.js index 2a4ec2fa..c35be792 100644 --- a/visualpython/js/m_ml/DimensionReduction.js +++ b/visualpython/js/m_ml/DimensionReduction.js @@ -87,7 +87,11 @@ define([ let config = that.modelConfig[that.state.modelType]; if (config && config.install != undefined) { // insert install code - com_interface.insertCell('code', config.install, true, 'Machine Learning > DimensionReduction'); + let installCode = config.install; + if (vpConfig.extensionType === 'lite') { + installCode = installCode.replace('!', '%'); + } + com_interface.insertCell('code', installCode, true, 'Machine Learning > DimensionReduction'); } }); diff --git a/visualpython/js/m_ml/FitPredict.js b/visualpython/js/m_ml/FitPredict.js index 8cf91de5..d48c20c5 100644 --- a/visualpython/js/m_ml/FitPredict.js +++ b/visualpython/js/m_ml/FitPredict.js @@ -41,6 +41,8 @@ define([ method: '', action: {}, optionConfig: {}, + modelEditorType: '', + modelEditorName: '', userOption: '', ...this.state } @@ -315,7 +317,7 @@ define([ } generateCode() { - let { model } = this.state; + let { model, modelType, modelEditorName } = this.state; let code = new com_String(); let replaceDict = {'${model}': model}; @@ -336,6 +338,31 @@ define([ code.appendLine(); code.append(allocateCode); } + // Data Preparation > Scaling + const scalingTypeList = ['StandardScaler', 'RobustScaler', 'MinMaxScaler', 'Normalizer']; + // Dimension Reduction + const dimensionTypeList = ['PCA', 'NMF']; + if (scalingTypeList.includes(modelType) || dimensionTypeList.includes(modelType)) { + // fit_transform, transform returns df_trans also + switch (modelEditorName) { + case 'fit_transform': + const allocatedFitTrans = this.state.fit_trans_allocate || 'trans'; + code.appendLine(); + code.appendLine(); + code.appendFormatLine("df_{0} = pd.DataFrame({1}, columns=[{2}])", allocatedFitTrans, allocatedFitTrans, this.state.fit_trans_featureData); + code.append("df_" + allocatedFitTrans); + break; + case 'transform': + const allocatedTrans = this.state.trans_allocate || 'trans'; + code.appendLine(); + code.appendLine(); + code.appendFormatLine("df_{0} = pd.DataFrame({1}, columns=[{2}])", allocatedTrans, allocatedTrans, this.state.trans_featureData); + code.append("df_" + allocatedTrans); + break; + default: + break; + } + } } return code.toString(); @@ -409,7 +436,7 @@ define([ code: '${model}.fit(${fit_featureData})', description: 'Fit Encoder/Scaler to X.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' } + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' } ] }, 'fit_transform': { @@ -418,16 +445,21 @@ define([ code: '${fit_trans_allocate} = ${model}.fit_transform(${fit_trans_featureData})', description: 'Fit Encoder/Scaler to X, then transform X.', options: [ - { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'fit_trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] }, 'transform': { - ...defaultActions['transform'], - description: 'Transform labels to normalized encoding.' + name: 'transform', + label: 'Transform', + code: '${trans_allocate} = ${model}.transform(${trans_featureData})', + description: 'Transform labels to normalized encoding.', + options: [ + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, + { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } + ] } } - if (modelType != 'ColumnTransformer') { actions = { ...actions, @@ -437,12 +469,75 @@ define([ code: '${inverse_allocate} = ${model}.inverse_transform(${inverse_featureData})', description: 'Transform binary labels back to multi-class labels.', options: [ - { name: 'inverse_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'inverse_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'inverse_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'inv_trans' } ] } } } + if (modelType == 'LabelEncoder') { + actions = { + ...actions, + 'fit': { + name: 'fit', + label: 'Fit', + code: '${model}.fit(${fit_featureData})', + description: 'Fit Encoder/Scaler to X.', + options: [ + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', + columnSelection: 'single', returnFrameType: 'Series' } + ] + }, + 'fit_transform': { + name: 'fit_transform', + label: 'Fit and transform', + code: '${fit_trans_allocate} = ${model}.fit_transform(${fit_trans_featureData})', + description: 'Fit Encoder/Scaler to X, then transform X.', + options: [ + { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', + columnSelection: 'single', returnFrameType: 'Series' }, + { name: 'fit_trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } + ] + }, + 'transform': { + name: 'transform', + label: 'Transform', + code: '${trans_allocate} = ${model}.transform(${trans_featureData})', + description: 'Transform labels to normalized encoding.', + options: [ + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', + columnSelection: 'single', returnFrameType: 'Series' }, + { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } + ] + } + } + } + if (modelType === 'SMOTE') { + actions = { + 'fit': { + name: 'fit', + label: 'Fit', + code: '${model}.fit(${fit_featureData}, ${fit_targetData})', + description: 'Check inputs and statistics of the sampler.', + options: [ + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'fit_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' } + ] + }, + 'fit_resample': { + name: 'fit_resample', + label: 'Fit and resample', + code: '${fit_res_allocateX}, ${fit_res_allocatey} = ${model}.fit_resample(${fit_res_featureData}, ${fit_res_targetData})', + description: 'Resample the dataset.', + options: [ + { name: 'fit_res_allocateX', label: 'Allocate feature', component: ['input'], placeholder: 'New variable', value: 'X_res' }, + { name: 'fit_res_allocatey', label: 'Allocate target', component: ['input'], placeholder: 'New variable', value: 'y_res' }, + { name: 'fit_res_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'fit_res_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' } + ] + } + } + } break; case 'Regression': actions = { @@ -581,7 +676,7 @@ define([ code: '${model}.fit(${fit_featureData})', description: 'Fit X into an embedded space.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' } + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' } ] }, 'fit_transform': { @@ -590,7 +685,7 @@ define([ code: '${fit_trans_allocate} = ${model}.fit_transform(${fit_trans_featureData})', description: 'Fit X into an embedded space and return that transformed output.', options: [ - { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'fit_trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] } @@ -605,7 +700,7 @@ define([ code: '${model}.fit(${fit_featureData}, ${fit_targetData})', description: 'Fit the Linear Discriminant Analysis model.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'fit_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y' } ] }, @@ -615,7 +710,7 @@ define([ code: '${fit_trans_allocate} = ${model}.fit_transform(${fit_trans_featureData}${fit_trans_targetData})', description: 'Fit to data, then transform it.', options: [ - { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'fit_trans_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y' }, { name: 'fit_trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] @@ -626,7 +721,7 @@ define([ code: '${pred_allocate} = ${model}.predict(${pred_featureData})', description: 'Predict class labels for samples in X.', options: [ - { name: 'pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'pred_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'pred' } ] }, @@ -636,7 +731,7 @@ define([ code: '${trans_allocate} = ${model}.transform(${trans_featureData})', description: 'Project data to maximize class separation.', options: [ - { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] } @@ -650,7 +745,7 @@ define([ code: '${model}.fit(${fit_featureData})', description: 'Fit X into an embedded space.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' } + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' } ] }, 'fit_transform': { @@ -659,7 +754,7 @@ define([ code: '${fit_trans_allocate} = ${model}.fit_transform(${fit_trans_featureData})', description: 'Fit the model with X and apply the dimensionality reduction on X.', options: [ - { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'fit_trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] }, @@ -669,7 +764,7 @@ define([ code: '${inverse_allocate} = ${model}.inverse_transform(${inverse_featureData})', description: 'Transform data back to its original space.', options: [ - { name: 'inverse_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'inverse_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'inverse_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'inv_trans' } ] }, @@ -679,12 +774,58 @@ define([ code: '${trans_allocate} = ${model}.transform(${trans_featureData})', description: 'Apply dimensionality reduction to X.', options: [ - { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] } } break; + case 'ETC': + if (modelType === 'GridSearchCV') { + actions = { + 'fit': { + name: 'fit', + label: 'Fit', + code: '${model}.fit(${fit_featureData}${fit_targetData})', + description: 'Run fit with all sets of parameters.', + options: [ + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'fit_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train', usePair: true, pairKey: 'y' } + ] + }, + 'predict': { + name: 'predict', + label: 'Predict', + code: '${pred_allocate} = ${model}.predict(${pred_featureData})', + description: 'Call predict on the estimator with the best found parameters.', + options: [ + { name: 'pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_test' }, + { name: 'pred_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'pred' } + ] + }, + 'inverse_transform': { + name: 'inverse_transform', + label: 'Inverse transform', + code: '${inverse_allocate} = ${model}.inverse_transform(${inverse_featureData})', + description: 'Call inverse_transform on the estimator with the best found params.', + options: [ + { name: 'inverse_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'inverse_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'inv_trans' } + ] + }, + 'transform': { + name: 'transform', + label: 'Transform', + code: '${trans_allocate} = ${model}.transform(${trans_featureData})', + description: 'Call transform on the estimator with the best found parameters.', + options: [ + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } + ] + } + } + } + break; } return actions; } diff --git a/visualpython/js/m_ml/GridSearch.js b/visualpython/js/m_ml/GridSearch.js new file mode 100644 index 00000000..1dfdbaa4 --- /dev/null +++ b/visualpython/js/m_ml/GridSearch.js @@ -0,0 +1,564 @@ +/* + * Project Name : Visual Python + * Description : GUI-based Python code generator + * File Name : GridSearch.js + * Author : Black Logic + * Note : GridSearch + * License : GNU GPLv3 with Visual Python special exception + * Date : 2023. 08. 09 + * Change Date : + */ + +//============================================================================ +// [CLASS] GridSearch +//============================================================================ +define([ + __VP_TEXT_LOADER__('vp_base/html/m_ml/gridSearch.html'), // INTEGRATION: unified version of text loader + __VP_CSS_LOADER__('vp_base/css/m_ml/gridSearch'), + 'vp_base/js/com/com_util', + 'vp_base/js/com/com_interface', + 'vp_base/js/com/com_String', + 'vp_base/js/com/com_generatorV2', + 'vp_base/data/m_ml/mlLibrary', + 'vp_base/js/com/component/PopupComponent', + 'vp_base/js/com/component/MultiSelector', + 'vp_base/js/com/component/SuggestInput', + 'vp_base/js/com/component/ModelEditor' +], function(msHtml, msCss, com_util, com_interface, com_String, com_generator, ML_LIBRARIES, PopupComponent, MultiSelector, SuggestInput, ModelEditor) { + + /** + * GridSearch + */ + class GridSearch extends PopupComponent { + _init() { + super._init(); + this.config.sizeLevel = 3; + this.config.dataview = false; + this.config.docs = 'https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.GridSearchCV.html'; + + this.state = { + modelType: 'ln-rgs', + userOption: '', + allocateToCreation: 'model', + ...this.state + } + + this.targetSetTag = null; + + this.modelConfig = ML_LIBRARIES; + + this.modelTypeList = { + 'Regression': ['ln-rgs', 'ridge', 'lasso', 'elasticnet', 'sv-rgs', 'dt-rgs', 'rf-rgs', 'gbm-rgs', 'xgb-rgs', 'lgbm-rgs', 'cb-rgs'], + 'Classification': ['lg-rgs', 'bern-nb', 'mulnom-nb', 'gaus-nb', 'sv-clf', 'dt-clf', 'rf-clf', 'gbm-clf', 'xgb-clf', 'lgbm-clf', 'cb-clf'] + } + + + } + + _unbindEvent() { + super._unbindEvent(); + $(document).off('click', this.wrapSelector('.vp-param-set-del')); + $(document).off('click', this.wrapSelector('.vp-param-item-add')); + $(document).off('click', this.wrapSelector('.vp-param-item-del')); + $(document).off('keyup', this.wrapSelector('.vp-param-val')); + $(document).off('click', this.wrapSelector('.vp-param-result-item-del')); + } + + _bindEvent() { + super._bindEvent(); + /** Implement binding events */ + var that = this; + + // select model type + $(this.wrapSelector('#modelType')).on('change', function() { + let modelType = $(this).val(); + that.state.modelType = modelType; + + // show install button + if (that.modelConfig[modelType].install != undefined) { + that.showInstallButton(); + } else { + that.hideInstallButton(); + } + + that.handleScoringOptions(modelType); + + // reset model param set + $(that.wrapSelector('.vp-param-grid-box')).html(''); + $(that.wrapSelector('.vp-param-grid-box')).html(that.templateForParamSet()); + }); + + // Add param set + $(this.wrapSelector('#vp_addParamSet')).on('click', function() { + let newSet = $(that.templateForParamSet()); + $(that.wrapSelector('.vp-param-grid-box')).append(newSet); + // focus + $(newSet)[0].scrollIntoView(); + }); + + // delete param set + $(document).on('click', this.wrapSelector('.vp-param-set-del'), function() { + $(this).closest('.vp-param-set-box').remove(); + + // rename param set + $(that.wrapSelector('.vp-param-set-name')).each((i, tag) => { + $(tag).text('Param set ' + (i + 1)); + }); + }); + + // Add param item + $(document).on('click', this.wrapSelector('.vp-param-item-add'), function() { + that.targetSetTag = $(this).parent(); // target param-set-box + that.openParamPopup(); + }); + + // Delete param item + $(document).on('click', this.wrapSelector('.vp-param-item-del'), function() { + $(this).closest('.vp-param-item').remove(); + }); + + // add param value - using enter + $(document).on('keyup', this.wrapSelector('.vp-param-val'), function(event) { + var keycode = event.keyCode + ? event.keyCode + : event.which; + if (keycode == vpEvent.keyManager.keyCode.enter) { // enter + that.handleAddParamValue($(this)); + } + if (keycode === 188) {// ,< + let val = $(this).val(); + $(this).val(val.split(',')[0]); // remove , and add param + that.handleAddParamValue($(this)); + } + }); + + // delete param set item + $(document).on('click', this.wrapSelector('.vp-param-result-item-del'), function() { + $(this).closest('.vp-param-result-item').remove(); + }); + } + + handleAddParamValue(thisTag) { + let parentTag = $(thisTag).parent(); + let paramIsText = $(parentTag).find('.vp-param-val').data('type') === 'text'; // text / var + let paramVal = $(parentTag).find('.vp-param-val').val(); + let reservedKeywordList = ['None', 'True', 'False', 'np.nan', 'np.NaN']; + if (reservedKeywordList.includes(paramVal)) { + paramIsText = false; + } + // check , and split it + let paramSplit = paramVal.split(','); + paramSplit && paramSplit.forEach(val => { + // add only if it is not empty value + if (val.trim() !== '') { + val = com_util.convertToStr(val.trim(), paramIsText); + $(parentTag).find('.vp-param-result-box').append(` + ${val} + + + `); + // clear param val + $(parentTag).find('.vp-param-val').val(''); + } + }); + } + + bindEventForInnerPopup() { + let that = this; + // Inner popup: Select param + $(document).off('click', this.wrapSelector('.vp-inner-param-list-item')); + $(document).on('click', this.wrapSelector('.vp-inner-param-list-item'), function() { + if ($(this).hasClass('selected')) { + $(this).removeClass('selected'); + } else { + $(this).addClass('selected'); + } + }); + + // Inner popup: Add param + $(this.wrapSelector('.vp-add-param-btn')).on('click', function() { + let newParam = $(that.wrapSelector('.vp-add-param-name')).val(); + if (newParam != undefined && newParam.trim() != '') { + // check if exist + let checkTag = $(that.wrapSelector(`.vp-inner-param-list-item[data-name="${newParam}"]`)); + if (checkTag.length > 0) { + if (!checkTag.hasClass('selected')) { + checkTag.addClass('selected'); + } + checkTag[0].scrollIntoView(); + } else { + // add to param list + $(that.wrapSelector('.vp-inner-param-list-box')).append( + `

${newParam}

` + ); + // scroll to added item + $(that.wrapSelector('.vp-inner-param-list-item.selected:last'))[0].scrollIntoView(); + } + + // clear input value + $(that.wrapSelector('.vp-add-param-name')).val(''); + } + }); + } + + handleScoringOptions(modelType) { + let options = { + 'Classification': [ + "'accuracy'", + "'balanced_accuracy'", + "'top_k_accuracy'", + "'average_precision'", + "'neg_brier_score'", + "'f1'", + "'f1_micro'", + "'f1_macro'", + "'f1_weighted'", + "'f1_samples'", + "'neg_log_loss'", + "'precision' etc.", + "'recall' etc.", + "'jaccard' etc.", + "'roc_auc'", + "'roc_auc_ovr'", + "'roc_auc_ovo'", + "'roc_auc_ovr_weighted'", + "'roc_auc_ovo_weighted'", + ], + 'Regression': [ + "'explained_variance'", + "'max_error'", + "'neg_mean_absolute_error'", + "'neg_mean_squared_error'", + "'neg_root_mean_squared_error'", + "'neg_mean_squared_log_error'", + "'neg_median_absolute_error'", + "'r2'", + "'neg_mean_poisson_deviance'", + "'neg_mean_gamma_deviance'", + "'neg_mean_absolute_percentage_error'", + "'d2_absolute_error_score'", + "'d2_pinball_score'", + "'d2_tweedie_score'" + ] + } + let modelCategory = this.modelTypeList['Regression'].includes(modelType)?'Regression':'Classification'; + + // Set suggestInput on scoring option + var suggestInput = new SuggestInput(); + suggestInput.setComponentID('scoring'); + suggestInput.setPlaceholder('Select option'); + suggestInput.addClass('vp-input vp-state'); + suggestInput.setSuggestList(function() { return options[modelCategory]; }); + suggestInput.setNormalFilter(true); + $(this.wrapSelector('#scoring')).replaceWith(suggestInput.toTagString()); + } + + templateForParamSet() { + let paramSetNo = 1; + // set param set number + paramSetNo += $(this.wrapSelector('.vp-param-set-box')).length; + return `

+ Param set ${paramSetNo} +

+ +

`; + } + + templateForParamPopup() { + let config = this.modelConfig[this.state.modelType]; + let optBox = new com_String(); + // render tag + config.options.forEach(opt => { + optBox.appendFormatLine('

{2}

' + , opt.name, opt.name, opt.name); + }); + + return ` +

NOTE: Select parameters to add.

+ ${optBox.toString()} +

+ + +

+ `; + } + + openParamPopup() { + let size = { width: 400, height: 300 }; + + $(this.wrapSelector('.vp-inner-popup-body')).empty(); + + // set size and position + $(this.wrapSelector('.vp-inner-popup-box')).css({ + width: size.width, + height: size.height, + left: 'calc(50% - ' + (size.width/2) + 'px)', + top: 'calc(50% - ' + (size.height/2) + 'px)', + }); + + $(this.wrapSelector('.vp-inner-popup-body')).html(this.templateForParamPopup()); + + this.bindEventForInnerPopup(); + + // show popup box + this.openInnerPopup('Add parameter'); + } + + handleInnerOk() { + // get selected param list + let paramList = []; + $(this.wrapSelector('.vp-inner-param-list-item.selected')).each((i, tag) => { + paramList.push($(tag).data('name')); + }); + + if (paramList.length > 0) { + // add param box + $(this.targetSetTag).find('.vp-param-item-box').append(this.templateForParamItemList(paramList)); + + this.closeInnerPopup(); + } else { + com_util.renderAlertModal('No params selected. Please select more than one param.'); + } + + this.targetSetTag = null; + } + + templateForBody() { + let page = $(msHtml); + + let that = this; + + //================================================================ + // Model creation + //================================================================ + // model types + let modelTypeTag = new com_String(); + Object.keys(this.modelTypeList).forEach(modelCategory => { + let modelOptionTag = new com_String(); + that.modelTypeList[modelCategory].forEach(opt => { + let optConfig = that.modelConfig[opt]; + let selectedFlag = ''; + if (opt == that.state.modelType) { + selectedFlag = 'selected'; + } + modelOptionTag.appendFormatLine('', + opt, selectedFlag, optConfig.name); + }) + modelTypeTag.appendFormatLine('{1}', + modelCategory, modelOptionTag.toString()); + }); + $(page).find('#modelType').html(modelTypeTag.toString()); + + //================================================================ + // GridSearch option + //================================================================ + $(page).find('.vp-model-option-box').html(this.templateForOption('grid-search', ['estimator', 'param_grid'])); + + // show install button + if (this.modelConfig[this.state.modelType].install != undefined) { + this.showInstallButton(); + } else { + this.hideInstallButton(); + } + + // render option page + // $(page).find('.vp-model-param-box').html(this.templateForOption(this.state.modelType)); + + return page; + } + + templateForOption(modelType, excludeOptList=[]) { + let config = this.modelConfig[modelType]; + let state = this.state; + + let optBox = new com_String(); + // render tag + config.options.forEach(opt => { + if (!excludeOptList.includes(opt.name)) { + optBox.appendFormatLine('{2}' + , opt.name, opt.name, com_util.optionToLabel(opt.name)); + let content = com_generator.renderContent(this, opt.component[0], opt, state); + optBox.appendLine(content[0].outerHTML); + } + }); + // render user option + optBox.appendFormatLine('{1}', 'userOption', 'User option'); + optBox.appendFormatLine('', + 'userOption', 'key=value, ...', this.state.userOption); + return optBox.toString(); + } + + /** + * template for param box + * @param {*} paramList ['param-name1', 'param-name2', ...] + */ + templateForParamItemList(paramList) { + let that = this; + let config = this.modelConfig[this.state.modelType]; + let paramSet = new com_String(); + paramList && paramList.forEach(name => { + // get option component + let componentType = 'input'; // default + let paramObj = config.options.filter(x => x.name === name).at(0); + if ((paramObj !== undefined) && (['option_select', 'bool_select'].includes(paramObj.component[0]))) { + componentType = 'option'; + } + paramSet.appendLine('

'); + paramSet.appendFormatLine('{1}', name, com_util.optionToLabel(name)); + paramSet.appendLine('

'); + paramSet.appendFormatLine('

', name); + paramSet.appendFormatLine('', name); + if (componentType === 'input') { + paramSet.appendFormatLine('', name); + + } else { + // paramSet.appendFormatLine('', name); + var suggestInput = new SuggestInput(); + suggestInput.setPlaceholder('Type here'); + suggestInput.addClass('vp-param-val'); + suggestInput.addAttribute('data-name', name); + suggestInput.addAttribute('data-type', (paramObj.type === 'text' ? 'text' : 'var')); + let optionList = paramObj.options; + if (paramObj.component[0] === 'bool_select') { + optionList = ['True', 'False']; + } + suggestInput.setSuggestList(function() { return optionList; }); + suggestInput.setNormalFilter(false); + suggestInput.setSelectEvent(function(selectedValue) { + // trigger change + let thisTag = $(that.wrapSelector('.' + suggestInput.uuid)); + that.handleAddParamValue($(thisTag)); + $(thisTag).val(''); + return false; + }); + paramSet.appendLine(suggestInput.toTagString()); + } + paramSet.appendLine('

'); + paramSet.appendLine('

'); // vp-param-item + }); + return paramSet.toString(); + } + + render() { + super.render(); + + // Model Editor + this.modelEditor = new ModelEditor(this, "model", "instanceEditor"); + + this.handleScoringOptions(this.state.modelType); + } + + generateInstallCode() { + let installCode = this.modelConfig[this.state.modelType].install; + if (vpConfig.extensionType === 'lite') { + installCode = installCode.replace('!', '%'); + } + return [ installCode ]; + } + + checkBeforeRun() { + // if no param is registered, stop and show alert + if ($(this.wrapSelector('.vp-param-result-item')).length <= 0) { + com_util.renderAlertModal('No params added. Please add params.'); + return false; + } + + return true; + } + + generateCode() { + let { modelType, userOption, allocateToCreation, model } = this.state; + let code = new com_String(); + /** + * Model Creation + * --- + * from module import model_function + * model = Model(key=value, ...) + */ + let gsConfig = this.modelConfig['grid-search']; + + + let state = JSON.parse(JSON.stringify(this.state)); + let estConfig = this.modelConfig[this.state.modelType]; + let estimator = com_generator.vp_codeGenerator(null, estConfig, {}, ''); + state['estimator'] = estimator; + state['param_grid'] = '{}'; + + let paramGrid = []; + // generate param_grid + $(this.wrapSelector('.vp-param-set-box')).each((i, tag) => { + let paramSet = {}; + $(tag).find('.vp-param-result-box').each((j, resTag) => { + let paramName = $(resTag).data('name'); + let paramValList = []; + $(resTag).find('.vp-param-result-item').each((k, itemTag) => { + let val = $(itemTag).data('value'); + if (!isNaN(val)) { + // numeric string -> numeric + val = parseFloat(val); + } + paramValList.push(val); + }); + if (paramValList.length > 0) { + // Add only its result exists + paramSet[paramName] = paramValList; + } + }); + if (Object.keys(paramSet).length > 0) { + paramGrid.push(paramSet); + } + }); + + state['param_grid'] = this.paramStringify(paramGrid); + + // import code + code.appendLine(gsConfig.import); + code.appendLine(estConfig.import); + + // model code + let gsCode = gsConfig.code; + gsCode = com_generator.vp_codeGenerator(this, gsConfig, state, (userOption != ''? ', ' + userOption : '')); + code.appendLine(); + code.appendFormat('{0} = {1}', allocateToCreation, gsCode); + + return code.toString(); + } + + paramStringify(paramGridList=[]) { + let paramGridCode = new com_String(); + if (paramGridList.length > 1) { + paramGridCode.append('['); + } + paramGridList.forEach((paramSet, i) => { + if (i > 0) { + paramGridCode.append(', \n '); + } + paramGridCode.append('{'); + Object.keys(paramSet).forEach((paramName, j) => { + if (j > 0) { + paramGridCode.append(', '); + } + paramGridCode.appendFormat("'{0}': [{1}]", paramName, paramSet[paramName].toString()); + }); + paramGridCode.append('}'); + }) + + if (paramGridList.length > 1) { + paramGridCode.append(']'); + } + return paramGridCode.toString(); + } + + } + + return GridSearch; +}); \ No newline at end of file diff --git a/visualpython/js/m_ml/ModelInfo.js b/visualpython/js/m_ml/ModelInfo.js index 00cdcf4a..23bf679b 100644 --- a/visualpython/js/m_ml/ModelInfo.js +++ b/visualpython/js/m_ml/ModelInfo.js @@ -260,6 +260,10 @@ define([ that.config.checkModules = ['pd', 'vp_create_feature_importances']; } else if (name == 'plot_feature_importances') { that.config.checkModules = ['pd', 'plt', 'vp_create_feature_importances', 'vp_plot_feature_importances']; + } else if (name == 'permutation_importance') { + that.config.checkModules = ['pd', 'vp_create_permutation_importances']; + } else if (name == 'plot_permutation_importance') { + that.config.checkModules = ['pd', 'vp_create_permutation_importances', 'vp_plot_permutation_importances']; } else { that.config.checkModules = ['pd']; } @@ -409,16 +413,46 @@ define([ name: 'permutation_importance', label: 'Permutation importance', import: 'from sklearn.inspection import permutation_importance', - code: '${importance_allocate} = permutation_importance(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${random_state}${etc})', + code: '${importance_allocate} = vp_create_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort})', description: 'Permutation importance for feature evaluation.', options: [ { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, - { name: 'scoring', component: ['input'], usePair: true }, - { name: 'random_state', component: ['input_number'], placeholder: '123', usePair: true }, + { name: 'scoring', component: ['option_suggest'], usePair: true, type: 'text', + options: [ + 'explained_variance', 'max_error', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_root_mean_squared_error', + 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'r2', 'neg_mean_poisson_deviance', 'neg_mean_gamma_deviance', + 'neg_mean_absolute_percentage_error', + 'accuracy', 'balanced_accuracy', 'top_k_accuracy', 'average_precision', 'neg_brier_score', + 'f1', 'f1_micro', 'f1_macro', 'f1_weighted', 'f1_samples', 'neg_log_loss', 'precision', 'recall', 'jaccard', + 'roc_auc', 'roc_auc_ovr', 'roc_auc_ovo', 'roc_auc_ovr_weighted', 'roc_auc_ovo_weighted' + ] }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, { name: 'importance_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'importances' } ] }, + 'plot_permutation_importance': { + name: 'plot_permutation_importance', + label: 'Plot permutation importance', + import: 'from sklearn.inspection import permutation_importance', + code: 'vp_plot_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort}${top_count})', + description: 'Permutation importance for feature evaluation.', + options: [ + { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, + { name: 'scoring', component: ['option_suggest'], usePair: true, type: 'text', + options: [ + 'explained_variance', 'max_error', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_root_mean_squared_error', + 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'r2', 'neg_mean_poisson_deviance', 'neg_mean_gamma_deviance', + 'neg_mean_absolute_percentage_error', + 'accuracy', 'balanced_accuracy', 'top_k_accuracy', 'average_precision', 'neg_brier_score', + 'f1', 'f1_micro', 'f1_macro', 'f1_weighted', 'f1_samples', 'neg_log_loss', 'precision', 'recall', 'jaccard', + 'roc_auc', 'roc_auc_ovr', 'roc_auc_ovo', 'roc_auc_ovr_weighted', 'roc_auc_ovo_weighted' + ] }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, + { name: 'top_count', label: 'Top count', component: ['input_number'], min: 0, usePair: true } + ] + }, 'feature_importances': { name: 'feature_importances', label: 'Feature importances', @@ -514,6 +548,19 @@ define([ } } } + if (modelType === 'SMOTE') { + infos = { + 'get_feature_names_out': { + name: 'get_feature_names_out', + label: 'Get feature names', + code: '${feature_names_allocate} = ${model}.get_feature_names_out()', + description: 'Get output feature names for transformation.', + options: [ + { name: 'feature_names_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'features' } + ] + } + } + } infos = { ...infos, 'get_params': defaultInfos['get_params'] @@ -545,7 +592,44 @@ define([ { name: 'cvs_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'scores' } ] }, - 'permutation_importance': defaultInfos['permutation_importance'], + 'permutation_importance': { + name: 'permutation_importance', + label: 'Permutation importance', + import: 'from sklearn.inspection import permutation_importance', + code: '${importance_allocate} = vp_create_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort})', + description: 'Permutation importance for feature evaluation.', + options: [ + { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, + { name: 'scoring', component: ['option_suggest'], usePair: true, type: 'text', + options: [ + 'explained_variance', 'max_error', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_root_mean_squared_error', + 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'r2', 'neg_mean_poisson_deviance', 'neg_mean_gamma_deviance', + 'neg_mean_absolute_percentage_error' + ] }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, + { name: 'importance_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'importances' } + ] + }, + 'plot_permutation_importance': { + name: 'plot_permutation_importance', + label: 'Plot permutation importance', + import: 'from sklearn.inspection import permutation_importance', + code: 'vp_plot_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort}${top_count})', + description: 'Permutation importance for feature evaluation.', + options: [ + { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, + { name: 'scoring', component: ['option_suggest'], usePair: true, type: 'text', + options: [ + 'explained_variance', 'max_error', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_root_mean_squared_error', + 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'r2', 'neg_mean_poisson_deviance', 'neg_mean_gamma_deviance', + 'neg_mean_absolute_percentage_error' + ] }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, + { name: 'top_count', label: 'Top count', component: ['input_number'], min: 0, usePair: true } + ] + }, 'Coefficient': { name: 'coef_', label: 'Coefficient', @@ -631,7 +715,44 @@ define([ { name: 'auc_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_test' } ] }, - 'permutation_importance': defaultInfos['permutation_importance'] + 'permutation_importance': { + name: 'permutation_importance', + label: 'Permutation importance', + import: 'from sklearn.inspection import permutation_importance', + code: '${importance_allocate} = vp_create_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort})', + description: 'Permutation importance for feature evaluation.', + options: [ + { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, + { name: 'scoring', component: ['option_suggest'], usePair: true, type: 'text', + options: [ + 'accuracy', 'balanced_accuracy', 'top_k_accuracy', 'average_precision', 'neg_brier_score', + 'f1', 'f1_micro', 'f1_macro', 'f1_weighted', 'f1_samples', 'neg_log_loss', 'precision', 'recall', 'jaccard', + 'roc_auc', 'roc_auc_ovr', 'roc_auc_ovo', 'roc_auc_ovr_weighted', 'roc_auc_ovo_weighted' + ] }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, + { name: 'importance_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'importances' } + ] + }, + 'plot_permutation_importance': { + name: 'plot_permutation_importance', + label: 'Plot permutation importance', + import: 'from sklearn.inspection import permutation_importance', + code: 'vp_plot_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort}${top_count})', + description: 'Permutation importance for feature evaluation.', + options: [ + { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, + { name: 'scoring', component: ['option_suggest'], usePair: true, type: 'text', + options: [ + 'accuracy', 'balanced_accuracy', 'top_k_accuracy', 'average_precision', 'neg_brier_score', + 'f1', 'f1_micro', 'f1_macro', 'f1_weighted', 'f1_samples', 'neg_log_loss', 'precision', 'recall', 'jaccard', + 'roc_auc', 'roc_auc_ovr', 'roc_auc_ovo', 'roc_auc_ovr_weighted', 'roc_auc_ovo_weighted' + ] }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, + { name: 'top_count', label: 'Top count', component: ['input_number'], min: 0, usePair: true } + ] + }, } // feature importances @@ -779,6 +900,39 @@ define([ } } break; + case 'ETC': + if (modelType === 'GridSearchCV') { + infos = { + 'best_estimator_': { + name: 'best_estimator_', + label: 'Best estimator', + code: '${best_estimator_allocate} = ${model}.best_estimator_', + description: 'Estimator that was chosen by the search.', + options: [ + { name: 'best_estimator_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'best_estimator' } + ] + }, + 'best_score_': { + name: 'best_score_', + label: 'Best score', + code: '${best_score_allocate} = ${model}.best_score_', + description: 'Mean cross-validated score of the best_estimator.', + options: [ + { name: 'best_score_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'best_score' } + ] + }, + 'best_params_': { + name: 'best_params_', + label: 'Best params', + code: '${best_params_allocate} = ${model}.best_params_', + description: 'Parameter setting that gave the best results on the hold out data.', + options: [ + { name: 'best_params_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'best_params' } + ] + } + } + } + break; } return infos; } diff --git a/visualpython/js/m_ml/Pipeline.js b/visualpython/js/m_ml/Pipeline.js new file mode 100644 index 00000000..856c33db --- /dev/null +++ b/visualpython/js/m_ml/Pipeline.js @@ -0,0 +1,798 @@ +/* + * Project Name : Visual Python + * Description : GUI-based Python code generator + * File Name : Pipeline.js + * Author : Black Logic + * Note : Pipeline + * License : GNU GPLv3 with Visual Python special exception + * Date : 2023. 08. 09 + * Change Date : + */ + +//============================================================================ +// [CLASS] Pipeline +//============================================================================ +define([ + __VP_TEXT_LOADER__('vp_base/html/m_ml/pipeline.html'), // INTEGRATION: unified version of text loader + __VP_CSS_LOADER__('vp_base/css/m_ml/pipeline'), + __VP_RAW_LOADER__('vp_base/data/libraries.json'), + 'vp_base/js/com/com_util', + 'vp_base/js/com/com_String', + 'vp_base/js/com/com_generatorV2', + 'vp_base/data/m_ml/mlLibrary', + 'vp_base/js/com/component/PopupComponent', + 'vp_base/js/com/component/SuggestInput', + 'vp_base/js/com/component/ModelEditor' +], function(msHtml, msCss, librariesJson, com_util, com_String, com_generator, ML_LIBRARIES, PopupComponent, SuggestInput, ModelEditor) { + + /** + * Pipeline + */ + class Pipeline extends PopupComponent { + _init() { + super._init(); + this.config.sizeLevel = 4; + this.config.dataview = false; + this.config.autoScroll = false; + + this.state = { + templateType: '', + modelStep: -1, // get modelStep sequence + modelType: '', // get modelType + modelTypeName: '', // get modelTypeName + model: 'model', + // now selected pipeline info + pipeline: [ + // copy of step + info + // { name, label, useApp, state, app } + ], + ...this.state + } + + /** + * useApp + - Data Split + - Data Prep + - Regressor + - Classifier + - Clustering + - Dimension + - GridSearch + - Evaluation + + - Fit + - Transform + - Predict + - Fit and Predict + - Fit and Transform + - Fit and Resample + */ + this.templateList = { + 'data-prep': { + label: 'Data Preparation', + modelStep: 0, + step: [ + /** + * ml_* is pre-defined app + * pp_* is defined only for Pipeline + */ + { name: 'ml_dataPrep', label: 'Data Prep', useApp: true, child: ['pp_fit', 'pp_transform', 'pp_fit_resample'] }, + { name: 'pp_fit', label: 'Fit' }, + { name: 'pp_transform', label: 'Transform' }, + { name: 'pp_fit_resample', label: 'Fit and Resample' } + ] + }, + 'regression': { + label: 'Regression', + modelStep: 1, + step: [ + { name: 'ml_dataSplit', label: 'Data Split', useApp: true }, + { name: 'ml_regression', label: 'Regressor', useApp: true, child: ['pp_fit', 'pp_predict'] }, + { name: 'pp_fit', label: 'Fit' }, + { name: 'pp_predict', label: 'Predict' }, + { name: 'ml_evaluation', label: 'Evaluation', useApp: true, state: { modelType: 'rgs' } }, + ] + }, + 'classification': { + label: 'Classification', + modelStep: 1, + step: [ + { name: 'ml_dataSplit', label: 'Data Split', useApp: true }, + { name: 'ml_classification', label: 'Classifier', useApp: true, child: ['pp_fit', 'pp_predict'] }, + { name: 'pp_fit', label: 'Fit' }, + { name: 'pp_predict', label: 'Predict' }, + { name: 'ml_evaluation', label: 'Evaluation', useApp: true, state: { modelType: 'clf' } }, + ] + }, + 'clustering': { + label: 'Clustering', + modelStep: 0, + step: [ + { name: 'ml_clustering', label: 'Clustering', useApp: true, child: ['pp_fit', 'pp_predict', 'pp_fit_predict', 'pp_transform'] }, + { name: 'pp_fit', label: 'Fit' }, + { name: 'pp_predict', label: 'Predict' }, + { name: 'pp_fit_predict', label: 'Fit and Predict' }, + { name: 'pp_transform', label: 'Transform' }, + { name: 'ml_evaluation', label: 'Evaluation', useApp: true, state: { modelType: 'cls' } }, + ] + }, + 'dimension': { + label: 'Dimension Reduction', + modelStep: 0, + step: [ + { name: 'ml_dimensionReduction', label: 'Dimension Reduction', useApp: true, child: ['pp_fit', 'pp_transform', 'pp_fit_transform'] }, + { name: 'pp_fit', label: 'Fit' }, + { name: 'pp_transform', label: 'Transform' }, + { name: 'pp_fit_transform', label: 'Fit and Transform' }, + ] + }, + 'gridSearch': { + label: 'GridSearch', + modelStep: 1, + step: [ + { name: 'ml_dataSplit', label: 'Data Split', useApp: true }, + { name: 'ml_gridSearch', label: 'GridSearch', useApp: true, child: ['pp_fit', 'pp_predict'] }, + { name: 'pp_fit', label: 'Fit' }, + { name: 'pp_predict', label: 'Predict' }, + { name: 'ml_evaluation', label: 'Evaluation', useApp: true }, + ] + } + } + + // menu libraries for ml + let libObj = {}; + if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { + libObj = librariesJson; + + this.MlAppComponent = {}; + this.MlAppComponent['ml_dataSplit'] = require('./dataSplit'); + this.MlAppComponent['ml_dataPrep'] = require('./DataPrep'); + this.MlAppComponent['ml_regression'] = require('./Regression'); + this.MlAppComponent['ml_classification'] = require('./Classification'); + this.MlAppComponent['ml_clustering'] = require('./Clustering'); + this.MlAppComponent['ml_dimensionReduction'] = require('./DimensionReduction'); + this.MlAppComponent['ml_gridSearch'] = require('./GridSearch'); + this.MlAppComponent['ml_evaluation'] = require('./evaluation'); + } else { + libObj = JSON.parse(librariesJson); + } + this.mlAppList = libObj.library.item.filter(x => x.id === 'pkg_ml')[0].item; + + this.modelConfig = ML_LIBRARIES; + } + + _unbindEvent() { + super._unbindEvent(); + + $(document).off('click', this.wrapSelector('.vp-pp-item[data-flag="enabled"]')); + $(document).off('click', this.wrapSelector('.vp-pp-item-toggle')); + $(document).off('change', this.wrapSelector(`#modelType`)); + $(document).off('change', this.wrapSelector('#allocateToCreation')); + + } + + _bindEvent() { + super._bindEvent(); + /** Implement binding events */ + var that = this; + + // select template + $(this.wrapSelector('#templateType')).on('change', function() { + let type = $(this).val(); + that.state.templateType = type; + that.state.model = 'model'; + that.state.modelType = ''; + that.state.modelTypeName = ''; + that.state.modelStep = -1; + + // reset pp-page + $(that.wrapSelector('.vp-pp-template')).html(''); + $(that.wrapSelector('.vp-pp-step-content')).html(''); + $(that.wrapSelector('.vp-pp-step-title')).text(''); + that.state.pipeline = []; + $(that.wrapSelector('.vp-pp-step-prev')).addClass('disabled'); + $(that.wrapSelector('.vp-pp-step-next')).addClass('disabled'); + + that.handleChangeTemplate(type); + }); + + // select pipeline item + $(document).on('click', this.wrapSelector('.vp-pp-item[data-flag="enabled"]'), function() { + if (!$(this).hasClass('selected')) { + let title = $(this).data('label'); + let stepSeq = parseInt($(this).data('step')); // 0 ~ n + let name = $(this).data('name'); + let ppObj = that.state.pipeline[stepSeq]; + // set title + $(that.wrapSelector('.vp-pp-step-title')).text(title); + + // show page + $(that.wrapSelector(`.vp-pp-step-page:not([data-name="${name}"])`)).hide(); + $(that.wrapSelector(`.vp-pp-step-page[data-name="${name}"]`)).show(); + if (ppObj.useApp === true) { + ppObj.app && that.handleAppView(name, ppObj.app); + ppObj.app && ppObj.app.open($(that.wrapSelector(`.vp-pp-step-page[data-name="${name}"]`))); + } else { + that.renderApp(name); + } + + // check selected + $(that.wrapSelector('.vp-pp-item')).removeClass('selected'); + $(this).addClass('selected'); + + // check prev/next + let prevTagList = $(this).prevAll('.vp-pp-item[data-flag="enabled"]:visible'); + let nextTagList = $(this).nextAll('.vp-pp-item[data-flag="enabled"]:visible'); + if (prevTagList.length == 0) { + $(that.wrapSelector('.vp-pp-step-prev')).addClass('disabled'); + } else { + $(that.wrapSelector('.vp-pp-step-prev')).removeClass('disabled'); + } + if (nextTagList.length == 0) { + $(that.wrapSelector('.vp-pp-step-next')).addClass('disabled'); + } else { + $(that.wrapSelector('.vp-pp-step-next')).removeClass('disabled'); + } + } + }); + + // pipeline item toggle (enable/disable) + $(document).on('click', this.wrapSelector('.vp-pp-item-toggle'), function(evt) { + evt.stopPropagation(); + let itemTag = $(this).closest('.vp-pp-item'); + let name = $(itemTag).attr('data-name'); + let flag = $(itemTag).attr('data-flag'); // enabled / disabled + if (flag === 'enabled') { + $(itemTag).attr('data-flag', 'disabled'); + $(this).prop('checked', false); + } else if (flag === 'disabled') { + $(itemTag).attr('data-flag', 'enabled'); + $(this).prop('checked', true); + } + + if ($(itemTag).hasClass('selected')) { + // check if this page is this item's, then hide it. + $(that.wrapSelector('.vp-pp-step-title')).text(''); + $(that.wrapSelector(`.vp-pp-step-page[data-name="${name}"]`)).hide(); + $(itemTag).removeClass('selected'); + + // disable prev/next + $(that.wrapSelector('.vp-pp-step-prev')).addClass('disabled'); + $(that.wrapSelector('.vp-pp-step-next')).addClass('disabled'); + } else { + // check prev/next + let selectedTag = $(that.wrapSelector('.vp-pp-item.selected')); + let prevTagList = $(selectedTag).prevAll('.vp-pp-item[data-flag="enabled"]:visible'); + let nextTagList = $(selectedTag).nextAll('.vp-pp-item[data-flag="enabled"]:visible'); + if (prevTagList.length == 0) { + $(that.wrapSelector('.vp-pp-step-prev')).addClass('disabled'); + } else { + $(that.wrapSelector('.vp-pp-step-prev')).removeClass('disabled'); + } + if (nextTagList.length == 0) { + $(that.wrapSelector('.vp-pp-step-next')).addClass('disabled'); + } else { + $(that.wrapSelector('.vp-pp-step-next')).removeClass('disabled'); + } + } + + }); + + // model type change event + $(document).on('change', this.wrapSelector(`#modelType`), function() { + let name = $(this).closest('.vp-pp-step-page').data('name'); + let modelCreatorList = ['ml_dataPrep', 'ml_regression', 'ml_classification', 'ml_clustering', 'ml_dimensionReduction', 'ml_gridSearch']; + if (modelCreatorList.includes(name)) { + let modelType = $(this).val(); + let modelObj = that.modelConfig[modelType]; + let modelTypeName = modelObj.returnType; + + that.state.modelType = modelType; + that.state.modelTypeName = modelTypeName; + + // show fit / predict / transform depends on model selection + let defaultActions = ['fit', 'predict', 'transform', 'fit_predict', 'fit_transform', 'fit_resample']; + let actions = that.modelEditor.getAction(modelTypeName); + defaultActions.forEach(actKey => { + if (actions[actKey] === undefined) { + // if undefined, hide step + $(that.wrapSelector(`.vp-pp-item[data-name="pp_${actKey}"]`)).hide(); + } else { + $(that.wrapSelector(`.vp-pp-item[data-name="pp_${actKey}"]`)).show(); + if (actKey === 'fit_predict') { + if (actions['fit'] === undefined || actions['predict'] === undefined) { + $(that.wrapSelector(`.vp-pp-item[data-name="pp_fit"]`)).hide(); + $(that.wrapSelector(`.vp-pp-item[data-name="pp_predict"]`)).hide(); + } else { + $(that.wrapSelector(`.vp-pp-item[data-name="pp_${actKey}"]`)).hide(); + } + } else if (actKey === 'fit_transform') { + if (actions['fit'] === undefined || actions['transform'] === undefined) { + $(that.wrapSelector(`.vp-pp-item[data-name="pp_fit"]`)).hide(); + $(that.wrapSelector(`.vp-pp-item[data-name="pp_transform"]`)).hide(); + } else { + $(that.wrapSelector(`.vp-pp-item[data-name="pp_${actKey}"]`)).hide(); + } + } else if (actKey === 'fit_resample') { + // for SMOTE: show fit_resample only + $(that.wrapSelector(`.vp-pp-item[data-name="pp_fit"]`)).hide(); + $(that.wrapSelector(`.vp-pp-item[data-name="pp_transform"]`)).hide(); + } + } + $(that.wrapSelector('.vp-pp-item')).removeClass('vp-last-visible'); + $(that.wrapSelector('.vp-pp-item:visible:last')).addClass('vp-last-visible'); + }); + + // change evaluation page's default selection + if (name === 'ml_gridSearch') { + let modelCategory = that.modelEditor.getModelCategory(modelTypeName); // Regression / Classification / Clustering + let evalModelTypeValue = 'rgs'; + switch (modelCategory) { + case 'Regression': + evalModelTypeValue = 'rgs'; + break; + case 'Classification': + evalModelTypeValue = 'clf'; + break; + case 'Clustering': + evalModelTypeValue = 'cls'; + break; + } + let evalPageTag = $(that.wrapSelector('.vp-pp-step-page[data-name="ml_evaluation"]')); + if (evalPageTag.is(':empty') === true) { + // evaluation page is not opened yet + // set state for evaluation page + let stepSeq = $(evalPageTag).data('step'); + let ppObj = that.state.pipeline[stepSeq]; + if (ppObj.app) { + ppObj.app.state.modelType = evalModelTypeValue; + } + } else { + let evalModelTypeTag = $(evalPageTag).find('#modelType'); + $(evalModelTypeTag).val(evalModelTypeValue); + $(evalModelTypeTag).trigger('change'); + } + } + } + }); + + // model allocation variable change + $(document).on('change', this.wrapSelector('#allocateToCreation'), function() { + let name = $(this).closest('.vp-pp-step-page').data('name'); + let modelAllocation = $(this).val(); + that.state.model = modelAllocation; + }); + + // click prev button + $(this.wrapSelector('.vp-pp-step-prev:not(.disabled)')).on('click', function() { + let selectedTag = $(that.wrapSelector('.vp-pp-item.selected')); + let prevTagList = $(selectedTag).prevAll('.vp-pp-item[data-flag="enabled"]:visible'); + $(prevTagList[0]).trigger('click'); + }); + + // click next button + $(this.wrapSelector('.vp-pp-step-next:not(.disabled)')).on('click', function() { + let selectedTag = $(that.wrapSelector('.vp-pp-item.selected')); + let nextTagList = $(selectedTag).nextAll('.vp-pp-item[data-flag="enabled"]:visible'); + $(nextTagList[0]).trigger('click'); + }); + } + + /** + * + * @param {*} type template type + */ + handleChangeTemplate(type) { + let that = this; + if (type !== '') { + let tplObj = this.templateList[type]; + this.state.modelStep = tplObj.modelStep; + + let ppTag = new com_String(); + let appFileList = []; + // load pipeline items + tplObj.step.forEach((stepObj, idx) => { + let { name, label, useApp=false, child=[], state={} } = stepObj; + ppTag.appendFormatLine(`

+ {3} + +

`, name, idx, label, label); + + // get pages + if (useApp === true) { + let mlObj = that.mlAppList.filter(x => x.id === name)[0]; + if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { + appFileList.push({ index: idx, name: name, file: 'vp_base/js/' + mlObj.file}); + } else { + appFileList.push({ index: idx, name: name, file: 'vp_base/js/' + mlObj.file}); + } + } + let pipeObj = { + name: name, + label: label, + useApp: useApp, + state: state + }; + if (tplObj.modelStep === idx) { + pipeObj.modelStep = true; + pipeObj.child = child; + } + that.state.pipeline.push(pipeObj); + // append pages + $(that.wrapSelector('.vp-pp-step-content')).append( + `

`); + if (useApp === false) { + that.renderApp(name); + // hide + $(that.wrapSelector(`.vp-pp-step-page[data-name="${name}"]`)).hide(); + } + + }); + $(that.wrapSelector('.vp-pp-template')).html(ppTag.toString()); + + // render pages + // for lite and lab + if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { + appFileList.forEach((obj, argIdx) => { + let MlComponent = that.MlAppComponent[obj.name]; + if (MlComponent) { + // DUP AREA: pp-1 + let { name, index, file } = obj; + let mlComponent = new MlComponent({ + config: { id: name, name: that.state.pipeline[index].label, path: file, category: 'Pipeline', resizable: false }, + ...that.state.pipeline[index].state + }); + mlComponent.loadState(); + // mlComponent.open($(that.wrapSelector(`.vp-pp-step-page[data-name="${appId}"]`))); + that.state.pipeline[index].app = mlComponent; + + if (that.state.pipeline[index].modelStep === true) { + // set default model type + that.state.model = mlComponent.state.allocateToCreation; + that.state.modelType = mlComponent.state.modelType; + let modelObj = that.modelConfig[that.state.modelType]; + that.state.modelTypeName = modelObj.code.split('(')[0]; + + that.state.pipeline[index].child.forEach(childId => { + that.renderApp(childId); + }); + } + // handle app view + that.handleAppView(name, mlComponent); + + // select first step + $(that.wrapSelector('.vp-pp-item[data-step="0"]')).click(); + $(that.wrapSelector('#modelType')).trigger('change'); + // end of DUP AREA: pp-1 + } + }); + + } else { + // for notebook and others + window.require(appFileList.map(x => x.file), function() { + appFileList.forEach((obj, argIdx) => { + let MlComponent = arguments[argIdx]; + if (MlComponent) { + // DUP AREA: pp-1 + let { name, index, file } = obj; + let mlComponent = new MlComponent({ + config: { id: name, name: that.state.pipeline[index].label, path: file, category: 'Pipeline', resizable: false }, + ...that.state.pipeline[index].state + }); + mlComponent.loadState(); + // mlComponent.open($(that.wrapSelector(`.vp-pp-step-page[data-name="${appId}"]`))); + that.state.pipeline[index].app = mlComponent; + + if (that.state.pipeline[index].modelStep === true) { + // set default model type + that.state.model = mlComponent.state.allocateToCreation; + that.state.modelType = mlComponent.state.modelType; + let modelObj = that.modelConfig[that.state.modelType]; + that.state.modelTypeName = modelObj.code.split('(')[0]; + + that.state.pipeline[index].child.forEach(childId => { + that.renderApp(childId); + }); + } + // handle app view + that.handleAppView(name, mlComponent); + + // select first step + $(that.wrapSelector('.vp-pp-item[data-step="0"]')).click(); + $(that.wrapSelector('#modelType')).trigger('change'); + // end of DUP AREA: pp-1 + } + }) + + }); + } + } + } + + templateForBody() { + let page = $(msHtml); + + let that = this; + + //================================================================ + // Template list creation + //================================================================ + let tplTag = new com_String(); + tplTag.appendLine(''); + Object.keys(this.templateList).forEach(tplKey => { + let tplObj = that.templateList[tplKey]; + let selectedFlag = ''; + if (tplKey == that.state.templateType) { + selectedFlag = 'selected'; + } + tplTag.appendFormatLine('', + tplKey, selectedFlag, tplObj.label); + }); + $(page).find('#templateType').html(tplTag.toString()); + + return page; + } + + render() { + super.render(); + + // resize codeview + $(this.wrapSelector('.vp-popup-codeview-box')).css({'height': '300px'}); + + // Model Editor + this.modelEditor = new ModelEditor(this, "model", "instanceEditor"); + + // load state + if (this.state.templateType !== '') { + $(this.wrapSelector('#templateType')).trigger('change'); + } else { + $(this.wrapSelector('.vp-pp-step-prev')).addClass('disabled'); + $(this.wrapSelector('.vp-pp-step-next')).addClass('disabled'); + } + } + + /** + * Handle app view before open + * @param {string} appId + * @param {*} mlApp + */ + handleAppView(appId, mlApp) { + switch (appId) { + case 'ml_dataSplit': + $(mlApp.wrapSelector('#inputData')).parent().hide(); + break; + case 'ml_evaluation': + // for pipeline + $(mlApp.wrapSelector('.vp-upper-box')).hide(); + $(mlApp.wrapSelector('.vp-upper-box.' + mlApp.state.modelType)).show(); + + $(mlApp.wrapSelector('.vp-eval-box')).hide(); + $(mlApp.wrapSelector('.vp-eval-' + mlApp.state.modelType)).show(); + + if (mlApp.state.modelType == 'rgs') { + // Regression + + } else if (mlApp.state.modelType == 'clf') { + // Classification + // if (this.state.roc_curve == false && this.state.auc == false) { + // $(page).find('.vp-ev-model.roc-auc').prop('disabled', true); + // } + } else { + // Clustering + if (mlApp.state.silhouetteScore == false) { + $(mlApp.wrapSelector('.vp-ev-model.silhouette')).prop('disabled', true); + } + if (mlApp.state.ari == false && mlApp.state.nmi == false) { + $(mlApp.wrapSelector('.vp-ev-model.ari-nmi')).prop('disabled', true); + } + } + break; + } + } + + /** + * Custom pages + * @param {*} appId + */ + renderApp(appId) { + let actions = this.modelEditor.getAction(this.state.modelTypeName); + let tag = ''; + switch (appId) { + case 'pp_fit': + tag = this.templateForOptionPage(actions['fit']); + break; + case 'pp_predict': + tag = this.templateForOptionPage(actions['predict']); + break; + case 'pp_transform': + tag = this.templateForOptionPage(actions['transform']); + break; + case 'pp_fit_predict': + tag = this.templateForOptionPage(actions['fit_predict']); + break; + case 'pp_fit_transform': + tag = this.templateForOptionPage(actions['fit_transform']); + break; + case 'pp_fit_resample': + tag = this.templateForOptionPage(actions['fit_resample']); + break; + } + $(this.wrapSelector(`.vp-pp-step-page[data-name="${appId}"]`)).html(` +

${tag}

+ `); + } + + templateForOptionPage(packageObj) { + let optBox = new com_String(); + // render tag + packageObj && packageObj.options && packageObj.options.forEach(opt => { + let label = opt.name; + if (opt.label != undefined) { + label = opt.label; + } + // fix label + label = com_util.optionToLabel(label); + optBox.appendFormatLine('{2}' + , opt.name, opt.name, label); + let tmpState = {}; + if (opt.value && opt.value !== '') { + tmpState[opt.name] = opt.value; + } + let content = com_generator.renderContent(this, opt.component[0], opt, tmpState); + optBox.appendLine(content[0].outerHTML); + }); + return optBox.toString(); + } + + checkBeforeRun() { + let that = this; + var result = true; + for (let idx = 0; idx < this.state.pipeline.length; idx++) { + let ppObj = this.state.pipeline[idx]; + var { name, label, useApp, app } = ppObj; + let requiredList = []; + result = true; + let isVisible = $(that.wrapSelector(`.vp-pp-item[data-step="${idx}"]`)).is(':visible') === true; + let isEnabled = $(that.wrapSelector(`.vp-pp-item[data-step="${idx}"]`)).attr('data-flag') === 'enabled'; + if (isVisible && isEnabled) { + switch (name) { + case 'ml_dataSplit': + requiredList = ['featureData', 'targetData']; + // check required data + for (let i = 0; i < requiredList.length; i++) { + let reqKey = requiredList[i]; + result = that._checkIsEmpty($(app.wrapSelector('#' + reqKey))); + if (result === false) { + // show page and focus it + $(that.wrapSelector(`.vp-pp-item[data-name="${name}"]`)).click(); + $(app.wrapSelector('#' + reqKey)).focus(); + break; + } + } + break; + case 'ml_gridSearch': + result = app.checkBeforeRun(); + if (result === false) { + // show page + $(that.wrapSelector(`.vp-pp-item[data-name="${name}"]`)).click(); + break; + } + break; + } + } + if (result === false) { + break; + } + } + return result; + + } + + _checkIsEmpty(tag) { + let requiredFilled = true; + // if it's empty, focus on it + if (tag && $(tag) && $(tag).val() == '') { + requiredFilled = false; + } + return requiredFilled; + } + + generateCodeForOptionPage(appId) { + let actions = this.modelEditor.getAction(this.state.modelTypeName); + let actObj = {}; + switch (appId) { + case 'pp_fit': + actObj = actions['fit']; + break; + case 'pp_predict': + actObj = actions['predict']; + break; + case 'pp_transform': + actObj = actions['transform']; + break; + case 'pp_fit_predict': + actObj = actions['fit_predict']; + break; + case 'pp_fit_transform': + actObj = actions['fit_transform']; + break; + case 'pp_fit_resample': + actObj = actions['fit_resample']; + break; + } + + let code = new com_String(); + if (actObj.import != undefined) { + code.appendLine(actObj.import); + code.appendLine(); + } + + let that = this; + let state = { + model: this.state.model + }; + $(this.wrapSelector(`.vp-pp-step-page[data-name="${appId}"] .vp-state`)).each((idx, tag) => { + let id = tag.id; + let value = that._getTagValue(tag); + state[id] = value; + }); + + let modelCode = com_generator.vp_codeGenerator(this, actObj, state); + modelCode = modelCode.replace('${model}', state.model); + code.append(modelCode); + return { state: state, code: code.toString() }; + } + + generateCode() { + let that = this; + let { template } = this.state; + let code = new com_String(); + + let stepNo = 1; + this.state.pipeline.forEach((ppObj, idx) => { + let { name, label, useApp, app } = ppObj; + + // check disabled + let isVisible = $(that.wrapSelector(`.vp-pp-item[data-step="${idx}"]`)).is(':visible') === true; + let isEnabled = $(that.wrapSelector(`.vp-pp-item[data-step="${idx}"]`)).attr('data-flag') === 'enabled'; + if (isVisible && isEnabled) { + if (code.toString() !== '') { + code.appendLine(); + code.appendLine(); + } + if (useApp === true) { + let appCode = app.generateCode(); + if (appCode instanceof Array) { + appCode = appCode.join('\n'); + } + if (appCode && appCode.trim() !== '') { + code.appendFormatLine("# [{0}] {1}", stepNo++, label); + if (name === 'ml_evaluation') { + // import auto generate + code.appendLine(app.generateImportCode().join('\n')); + } + code.append(appCode); + } + // save state + that.state.pipeline[idx].state = app.state; + } else { + let ppResult = that.generateCodeForOptionPage(name); + if (ppResult && ppResult?.code?.trim() !== '') { + code.appendFormatLine("# [{0}] {1}", stepNo++, label); + code.append(ppResult.code); + } + // save state + that.state.pipeline[idx].state = ppResult.state; + } + } + }); + + return code.toString(); + } + + } + + return Pipeline; +}); \ No newline at end of file diff --git a/visualpython/js/m_ml/Regression.js b/visualpython/js/m_ml/Regression.js index b4fde181..3c21d077 100644 --- a/visualpython/js/m_ml/Regression.js +++ b/visualpython/js/m_ml/Regression.js @@ -88,7 +88,11 @@ define([ let config = that.modelConfig[that.state.modelType]; if (config && config.install != undefined) { // insert install code - com_interface.insertCell('code', config.install, true, 'Machine Learning > Regression'); + let installCode = config.install; + if (vpConfig.extensionType === 'lite') { + installCode = installCode.replace('!', '%'); + } + com_interface.insertCell('code', installCode, true, 'Machine Learning > Regression'); } }); diff --git a/visualpython/js/m_ml/SaveLoad.js b/visualpython/js/m_ml/SaveLoad.js new file mode 100644 index 00000000..454f468c --- /dev/null +++ b/visualpython/js/m_ml/SaveLoad.js @@ -0,0 +1,121 @@ +/* + * Project Name : Visual Python + * Description : GUI-based Python code generator + * File Name : SaveLoad.js + * Author : Black Logic + * Note : Save and load models + * License : GNU GPLv3 with Visual Python special exception + * Date : 2023. 03. 09 + * Change Date : + */ + +//============================================================================ +// [CLASS] DataSets +//============================================================================ +define([ + __VP_CSS_LOADER__('vp_base/css/m_ml/saveLoad'), + __VP_TEXT_LOADER__('vp_base/html/m_ml/saveLoad.html'), + 'vp_base/js/com/com_util', + 'vp_base/js/com/com_Const', + 'vp_base/js/com/com_String', + 'vp_base/js/com/com_generatorV2', + 'vp_base/js/com/component/PopupComponent', + 'vp_base/js/com/component/FileNavigation', + 'vp_base/data/m_ml/mlLibrary', +], function(slCss, slHTML, com_util, com_Const, com_String, com_generator, PopupComponent, FileNavigation, ML_LIBRARIES) { + + /** + * SaveLoad + */ + class SaveLoad extends PopupComponent { + _init() { + super._init(); + this.config.sizeLevel = 2; + this.config.dataview = false; + this.config.checkModules = ['joblib']; + + this.state = { + modelio: 'model_save', // model_save / model_load + target: '', + allocateTo: '', + userOption: '', + savePath: '', + loadPath: '', + ...this.state + } + + this.mlConfig = ML_LIBRARIES; + } + + _bindEvent() { + super._bindEvent(); + let that = this; + + // select model + $(this.wrapSelector('#modelio')).on('change', function() { + let modelio = $(this).val(); + that.state.modelio = modelio; + $(that.wrapSelector('.vp-modelio-option-box')).html(that.templateForOption(modelio)); + + $(that.wrapSelector('#userOption')).val(''); + }); + + // user option + $(this.wrapSelector('#userOption')).on('change', function() { + that.state.userOption = $(this).val(); + }) + } + + templateForBody() { + let page = $(slHTML); + + // render option page + $(page).find('.vp-modelio-option-box').html(this.templateForOption(this.state.modelio)); + + $(page).find('#modelio').val(this.state.modelio); + $(page).find('#userOption').val(this.state.userOption); + + return page; + } + + templateForOption(modelio) { + let config = this.mlConfig[modelio]; + let state = this.state; + + let optBox = new com_String(); + // render tag + config.options.forEach(opt => { + optBox.appendFormatLine('{2}' + , opt.name, opt.name, com_util.optionToLabel(opt.name)); + let content = com_generator.renderContent(this, opt.component[0], opt, state); + optBox.appendLine(content[0].outerHTML); + }); + + // render file navigation + + + // show user option + if (config.code.includes('${etc}')) { + // render user option + optBox.appendFormatLine('{1}', 'userOption', 'User option'); + optBox.appendFormatLine('', + 'userOption', 'key=value, ...', this.state.userOption); + } + return optBox.toString(); + } + + generateCode() { + let { modelio, userOption } = this.state; + let code = new com_String(); + if (userOption && userOption != '') { + userOption = ', ' + userOption; + } + + let modelCode = com_generator.vp_codeGenerator(this, this.mlConfig[modelio], this.state, userOption); + code.append(modelCode); + return code.toString(); + } + } + + return SaveLoad; +}); \ No newline at end of file diff --git a/visualpython/js/m_ml/dataSplit.js b/visualpython/js/m_ml/dataSplit.js index bfa2aac2..a8c9417f 100644 --- a/visualpython/js/m_ml/dataSplit.js +++ b/visualpython/js/m_ml/dataSplit.js @@ -175,7 +175,7 @@ define([ options.appendFormat(', shuffle={0}', shuffle); } if (shuffle != 'False' && stratify && stratify != '') { - options.appendFormat(', startify={0}', stratify); + options.appendFormat(', stratify={0}', stratify); } let code = new com_String(); diff --git a/visualpython/js/m_ml/evaluation.js b/visualpython/js/m_ml/evaluation.js index f911159d..ed63120c 100644 --- a/visualpython/js/m_ml/evaluation.js +++ b/visualpython/js/m_ml/evaluation.js @@ -33,7 +33,7 @@ define([ this.config.checkModules = ['metrics']; this.state = { - modelType: 'rgs', + modelType: 'rgs', // rgs / clf / cls predictData: 'pred', targetData: 'y_test', // regression @@ -156,37 +156,11 @@ define([ }); $(page).find('#targetData2').replaceWith(targetData2Selector.toTagString()); - // load state - let that = this; - Object.keys(this.state).forEach(key => { - let tag = $(page).find('#' + key); - let tagName = $(tag).prop('tagName'); // returns with UpperCase - let value = that.state[key]; - if (value == undefined) { - return; - } - switch(tagName) { - case 'INPUT': - let inputType = $(tag).prop('type'); - if (inputType == 'text' || inputType == 'number' || inputType == 'hidden') { - $(tag).val(value); - break; - } - if (inputType == 'checkbox') { - $(tag).prop('checked', value); - break; - } - break; - case 'TEXTAREA': - case 'SELECT': - default: - $(tag).val(value); - break; - } - }); - $(page).find('.vp-upper-box').hide(); $(page).find('.vp-upper-box.' + this.state.modelType).show(); + + $(page).find('.vp-eval-box').hide(); + $(page).find('.vp-eval-' + this.state.modelType).show(); if (this.state.modelType == 'rgs') { // Regression @@ -215,7 +189,7 @@ define([ generateCode() { let codeCells = [ - ...this.generateImportCode() // run import codes + // ...this.generateImportCode() // run import codes ]; let code = new com_String(); let { @@ -228,15 +202,20 @@ define([ sizeOfClusters, silhouetteScore, ari, nmi, clusteredIndex, featureData2, targetData2 } = this.state; + // add import code for display and Markdown + let needDisplay = false; + let needMarkdown = false; //==================================================================== - // Classfication + // Classification //==================================================================== if (modelType == 'clf') { if (confusion_matrix) { code = new com_String(); code.appendLine("# Confusion Matrix"); - code.appendFormat('pd.crosstab({0}, {1}, margins=True)', targetData, predictData); + code.appendLine("display(Markdown('### Confusion Matrix'))"); + code.appendFormat('display(pd.crosstab({0}, {1}, margins=True))', targetData, predictData); + needMarkdown = true; codeCells.push(code.toString()); } if (report) { @@ -248,25 +227,25 @@ define([ if (accuracy) { code = new com_String(); code.appendLine("# Accuracy"); - code.appendFormat('metrics.accuracy_score({0}, {1})', targetData, predictData); + code.appendFormat("print('Accuracy: {}'.format(metrics.accuracy_score({0}, {1})))", targetData, predictData); codeCells.push(code.toString()); } if (precision) { code = new com_String(); code.appendLine("# Precision"); - code.appendFormat("metrics.precision_score({0}, {1}, average='weighted')", targetData, predictData); + code.appendFormat("print('Precision: {}'.format(metrics.precision_score({0}, {1}, average='weighted')))", targetData, predictData); codeCells.push(code.toString()); } if (recall) { code = new com_String(); code.appendLine("# Recall"); - code.appendFormat("metrics.recall_score({0}, {1}, average='weighted')", targetData, predictData); + code.appendFormat("print('Recall: {}'.format(metrics.recall_score({0}, {1}, average='weighted')))", targetData, predictData); codeCells.push(code.toString()); } if (f1_score) { code = new com_String(); code.appendLine("# F1-score"); - code.appendFormat("metrics.f1_score({0}, {1}, average='weighted')", targetData, predictData); + code.appendFormat("print('F1-score: {}'.format(metrics.f1_score({0}, {1}, average='weighted')))", targetData, predictData); codeCells.push(code.toString()); } // if (roc_curve) { @@ -300,14 +279,14 @@ define([ // } if (r_squared) { code = new com_String(); - code.appendLine("# R square"); - code.appendFormat('metrics.r2_score({0}, {1})', targetData, predictData); + code.appendLine("# R squared"); + code.appendFormat("print('R squared: {}'.format(metrics.r2_score({0}, {1})))", targetData, predictData); codeCells.push(code.toString()); } if (mae) { code = new com_String(); code.appendLine("# MAE(Mean Absolute Error)"); - code.appendFormat('metrics.mean_absolute_error({0}, {1})', targetData, predictData); + code.appendFormat("print('MAE: {}'.format(metrics.mean_absolute_error({0}, {1})))", targetData, predictData); codeCells.push(code.toString()); } if (mape) { @@ -316,22 +295,24 @@ define([ code.appendLine('def MAPE(y_test, y_pred):'); code.appendLine(' return np.mean(np.abs((y_test - pred) / y_test)) * 100'); code.appendLine(); - code.appendFormat('MAPE({0}, {1})', targetData, predictData); + code.appendFormat("print('MAPE: {}'.format(MAPE({0}, {1})))", targetData, predictData); codeCells.push(code.toString()); } if (rmse) { code = new com_String(); code.appendLine("# RMSE(Root Mean Squared Error)"); - code.appendFormat('metrics.mean_squared_error({0}, {1})**0.5', targetData, predictData); + code.appendFormat("print('RMSE: {}'.format(metrics.mean_squared_error({0}, {1})**0.5))", targetData, predictData); codeCells.push(code.toString()); } if (scatter_plot) { code = new com_String(); code.appendLine('# Regression plot'); + code.appendLine("display(Markdown('### Regression plot'))"); code.appendFormatLine('plt.scatter({0}, {1})', targetData, predictData); code.appendFormatLine("plt.xlabel('{0}')", targetData); code.appendFormatLine("plt.ylabel('{0}')", predictData); code.append('plt.show()'); + needMarkdown = true; codeCells.push(code.toString()); } } @@ -346,23 +327,34 @@ define([ if (silhouetteScore) { code = new com_String(); code.appendLine("# Silhouette score"); - code.appendFormat("print(f'Silhouette score: {metrics.cluster.silhouette_score({0}, {1})}')", featureData2, clusteredIndex); + code.appendFormat("print('Silhouette score: {}'.format(metrics.cluster.silhouette_score({0}, {1})))", featureData2, clusteredIndex); codeCells.push(code.toString()); } if (ari) { code = new com_String(); code.appendLine("# ARI(Adjusted Rand score)"); - code.appendFormat("print(f'ARI: {metrics.cluster.adjusted_rand_score({0}, {1})}')", targetData2, clusteredIndex); + code.appendFormat("print('ARI: {}'.format(metrics.cluster.adjusted_rand_score({0}, {1})))", targetData2, clusteredIndex); codeCells.push(code.toString()); } if (nmi) { code = new com_String(); code.appendLine("# NMI(Normalized Mutual Info Score)"); - code.appendFormat("print(f'NM: {metrics.cluster.normalized_mutual_info_score({0}, {1})}')", targetData2, clusteredIndex); + code.appendFormat("print('NM: {}'.format(metrics.cluster.normalized_mutual_info_score({0}, {1})))", targetData2, clusteredIndex); codeCells.push(code.toString()); } } - // return as seperated cells + if (needMarkdown === true) { + codeCells = [ + "from IPython.display import display, Markdown", + ...codeCells + ]; + } else if (needDisplay === true) { + codeCells = [ + "from IPython.display import display", + ...codeCells + ]; + } + // return as separated cells return codeCells; } diff --git a/visualpython/js/m_stats/Anova.js b/visualpython/js/m_stats/Anova.js new file mode 100644 index 00000000..4d6f901f --- /dev/null +++ b/visualpython/js/m_stats/Anova.js @@ -0,0 +1,511 @@ +/* + * Project Name : Visual Python + * Description : GUI-based Python code generator + * File Name : Anova.js + * Author : Black Logic + * Note : ANOVA + * License : GNU GPLv3 with Visual Python special exception + * Date : 2023. 05. 24 + * Change Date : + */ + +//============================================================================ +// [CLASS] Anova +//============================================================================ +define([ + __VP_TEXT_LOADER__('vp_base/html/m_stats/anova.html'), + 'vp_base/js/com/com_util', + 'vp_base/js/com/com_Const', + 'vp_base/js/com/com_String', + 'vp_base/js/com/com_generatorV2', + 'vp_base/js/com/component/PopupComponent', + 'vp_base/js/com/component/DataSelector', + 'vp_base/js/m_apps/Subset' +], function(nmHTML, com_util, com_Const, com_String, com_generator, PopupComponent, DataSelector, Subset) { + + /** + * Anova + */ + class Anova extends PopupComponent { + _init() { + super._init(); + /** Write codes executed before rendering */ + this.config.sizeLevel = 2; + this.config.checkModules = ['pd', 'np', 'stats', 'vp_confidence_interval', 'vp_sem']; + this.config.docs = 'https://docs.scipy.org/doc/scipy/reference/'; + + this.state = { + testType: 'one-way', + data: '', + dataType: '', + depVar: '', + factor: '', + factorA: '', + factorB: '', + covariate: '', + sigLevel: 0.05, + // Post hoc analysis option + tukeyHSD: true, + tukey: false, + scheffe: false, + duncan: false, + bonferroni: false, + // Display option + statistics: true, + boxplot: true, + equalVariance: true, + interPlot: true, + ...this.state + }; + + this.columnBindList = ['depVar', 'factor', 'factorA', 'factorB', 'covariate']; + + this.tmpInstallCode = []; // install codes + + this.subsetEditor = {}; + } + + _bindEvent() { + super._bindEvent(); + /** Implement binding events */ + var that = this; + + $(this.wrapSelector('#testType')).on('change', function() { + let testType = $(this).val(); + that.state.testType = testType; + + $(that.wrapSelector('.vp-st-option')).hide(); + $(that.wrapSelector('.vp-st-option.' + testType)).show(); + + that.tmpInstallCode = []; + that.hideInstallButton(); + + if (testType === 'one-way' || testType === 'two-way') { + if (that.state.tukey || that.state.scheffe || that.state.duncan) { + // Add installation code + if (vpConfig.extensionType === 'lite') { + that.tmpInstallCode = ["%pip install scikit-posthocs"]; + } else { + that.tmpInstallCode = ["!pip install scikit-posthocs"]; + } + that.showInstallButton(); + } + } else if (testType === 'ancova') { + // Add installation code : # pip install pingouin + if (vpConfig.extensionType === 'lite') { + that.tmpInstallCode = ["%pip install pingouin"]; + } else { + that.tmpInstallCode = ["!pip install pingouin"]; + } + that.showInstallButton(); + } + }); + + $(this.wrapSelector('#data')).on('change', function() { + if (that.state.dataType === 'Series') { + // Series + that.columnBindList.forEach(id => { + $(that.wrapSelector('#' + id)).html(''); + $(that.wrapSelector('#' + id)).prop('disabled', true); + }); + } else { + // DataFrame + that.columnBindList.forEach(id => { + $(that.wrapSelector('#' + id)).prop('disabled', false); + }); + com_generator.vp_bindColumnSource(that, 'data', that.columnBindList, 'select', false, false); + } + }); + + $(this.wrapSelector('.vp-st-posthoc-box .vp-state')).on('change', function() { + let id = $(this)[0].id; + let checked = $(this).prop('checked') === true; + that.state[id] = checked; + let { testType, tukey, scheffe, duncan } = that.state; + if (testType === 'one-way' || testType === 'two-way') { + if (tukey || scheffe || duncan) { + // Add installation code + if (vpConfig.extensionType === 'lite') { + that.tmpInstallCode = ["%pip install scikit-posthocs"]; + } else { + that.tmpInstallCode = ["!pip install scikit-posthocs"]; + } + that.showInstallButton(); + } else { + that.hideInstallButton(); + } + } + }); + + $(this.wrapSelector('')) + } + + templateForBody() { + let page = $(nmHTML); + let that = this; + + let dataSelector = new DataSelector({ + pageThis: this, id: 'data', placeholder: 'Select data', required: true, boxClasses: 'vp-flex-gap5', + allowDataType: ['DataFrame'], withPopup: false, + finish: function(data, type) { + that.state.data = data; + that.state.dataType = type; + $(that.wrapSelector('#data')).trigger('change'); + }, + select: function(data, type) { + that.state.data = data; + that.state.dataType = type; + $(that.wrapSelector('#data')).trigger('change'); + } + }); + $(page).find('#data').replaceWith(dataSelector.toTagString()); + + return page; + } + + render() { + super.render(); + let that = this; + + // render Subset + this.subsetEditor['data'] = new Subset({ + pandasObject: '', + config: { name: 'Subset', category: this.name } }, + { + useAsModule: true, + useInputColumns: true, + targetSelector: this.wrapSelector('#data'), + pageThis: this, + finish: function(code, state) { + that.state.data = code; + $(that.wrapSelector('#data')).val(code); + that.state.dataType = state.returnType; + $(that.wrapSelector('#data')).trigger('change'); + } + }); + + // bind column if data exist + if (this.state.data !== '') { + com_generator.vp_bindColumnSource(this, 'data', this.columnBindList, 'select', false, false); + } + + // control display option + $(this.wrapSelector('.vp-st-option')).hide(); + $(this.wrapSelector('.vp-st-option.' + this.state.testType)).show(); + } + + generateInstallCode() { + return this.tmpInstallCode; + } + + generateCode() { + let { + testType, data, depVar, factor, factorA, factorB, covariate, sigLevel, + // Post hoc analysis option + tukeyHSD, tukey, scheffe, duncan, bonferroni, + // Display option + statistics, boxplot, equalVariance, interPlot + } = this.state; + + // get only text without '' or "" + let depVarText = $(this.wrapSelector('#depVar option:selected')).text(); + let factorText = $(this.wrapSelector('#factor option:selected')).text(); + let factorAText = $(this.wrapSelector('#factorA option:selected')).text(); + let factorBText = $(this.wrapSelector('#factorB option:selected')).text(); + let covariateText = $(this.wrapSelector('#covariate option:selected')).text(); + + let codeList = []; + let code = new com_String(); + + // test type label + let testTypeLabel = $(this.wrapSelector('#testType option:selected')).text(); + code.appendFormatLine("# {0}", testTypeLabel); + code.appendFormat("vp_df = {0}.dropna().copy()", data); + + switch (testType) { + case 'one-way': + // 1. One-way ANOVA + code.appendLine(); + code.appendLine(); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("_df = pd.DataFrame()"); + code.appendFormatLine("for k, v in dict(list(vp_df.groupby({0})[{1}])).items():", factor, depVar); + code.appendLine(" _df_t = v.reset_index(drop=True)"); + code.appendLine(" _df_t.name = k"); + code.append(" _df = pd.concat([_df, _df_t], axis=1)"); + + // Display - Statistics + if (statistics === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Statistics"); + code.appendLine("display(Markdown('### Statistics'))"); + code.appendLine("display(pd.DataFrame(data={'Count':_df.count(),'Mean':_df.mean(numeric_only=True),'Std. Deviation':_df.std(numeric_only=True),'Min':_df.min(),'Max':_df.max(),"); + code.appendLine(" 'Std. Error Mean':_df.apply(vp_sem),'Confidence interval':0.95,"); + code.append(" 'Lower':_df.apply(vp_confidence_interval).T[0],'Upper':_df.apply(vp_confidence_interval).T[1] }))"); + } + // Display - Boxplot + if (boxplot === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Boxplot"); + code.appendLine("import seaborn as sns"); + code.appendLine("import warnings"); + code.appendLine("with warnings.catch_warnings():"); + code.appendLine(" warnings.simplefilter(action='ignore', category=Warning)"); + code.appendLine(" sns.boxplot(data=_df)"); + code.append(" plt.show()"); + } + // Display - Equal Variance + if (equalVariance === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Equal Variance test (Levene)"); + code.appendLine("from scipy import stats"); + code.appendLine("_lst = []"); + code.appendLine("_df.apply(lambda x: _lst.append(x.dropna()))"); + code.appendLine("_res = stats.levene(*_lst, center='mean')"); + code.appendLine("display(Markdown('### Equal Variance test (Levene)'))"); + code.append("display(pd.DataFrame(data={'Statistic':_res.statistic,'p-value':_res.pvalue}, index=['Equal Variance test (Levene)']))"); + } + + code.appendLine(); + code.appendLine(); + code.appendLine("# One-way ANOVA"); + code.appendLine("import statsmodels.formula.api as smf"); + code.appendLine("from statsmodels.stats.anova import anova_lm"); + code.appendFormatLine("_model = smf.ols('{0} ~ C({1})', vp_df)", depVarText, factorText); + code.appendLine("_result = _model.fit()"); + code.appendLine("_dfr = anova_lm(_result)"); + code.appendLine("_dfr.loc['Total','df'] = _dfr['df'].sum()"); + code.appendLine("_dfr.loc['Total','sum_sq'] = _dfr['sum_sq'].sum()"); + code.appendLine("display(Markdown('### One-way ANOVA'))"); + code.append("display(_dfr)"); + + // Post hoc analysis - Tukey HSD + if (tukeyHSD === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Post-hoc: Tukey HSD"); + code.appendLine("from statsmodels.sandbox.stats.multicomp import MultiComparison"); + code.appendFormatLine("_res = MultiComparison(vp_df[{0}], vp_df[{1}]).tukeyhsd(alpha={2})", depVar, factor, sigLevel); + code.appendLine("display(Markdown('### Post-hoc: Tukey HSD'))"); + code.append("display(_res.summary())"); + } + // Post hoc analysis - Bonferroni + if (bonferroni === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Post-hoc: Bonferroni"); + code.appendLine("from statsmodels.sandbox.stats.multicomp import MultiComparison"); + code.appendFormatLine("_res = MultiComparison(vp_df[{0}], vp_df[{1}]).allpairtest(stats.ttest_ind,alpha={2},method='bonf')", depVar, factor, sigLevel); + code.appendLine("display(Markdown('### Post-hoc: Bonferroni'))"); + code.append("display(_res[0])"); + } + + if (tukey === true || scheffe === true || duncan === true) { + // Post hoc analysis - Tukey + if (tukey === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Post-hoc: Tukey"); + code.appendLine("import scikit_posthocs as sph"); + code.appendLine("display(Markdown('### Post-hoc: Tukey'))"); + code.appendFormat("display(sph.posthoc_tukey(vp_df, val_col={0}, group_col={1}))", depVar, factor); + } + // Post hoc analysis - Scheffe + if (scheffe === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Post-hoc: Scheffe"); + code.appendLine("import scikit_posthocs as sph"); + code.appendLine("display(Markdown('### Post-hoc: Scheffe'))"); + code.appendFormat("display(sph.posthoc_scheffe(vp_df, val_col={0}, group_col={1}))", depVar, factor); + + } + // Post hoc analysis - duncan + if (duncan === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Post-hoc: Duncan"); + code.appendLine("import scikit_posthocs as sph"); + code.appendLine("display(Markdown('### Post-hoc: Duncan'))"); + code.appendFormat("display(sph.posthoc_dunn(vp_df, val_col={0}, group_col={1}))", depVar, factor); + } + } + + break; + case 'two-way': + // 1. Two-way ANOVA + code.appendLine(); + code.appendLine(); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("_df = pd.DataFrame()"); + code.appendFormatLine("for k, v in dict(list(vp_df.groupby([{0},{1}])[{2}])).items():", factorB, factorA, depVar); + code.appendLine(" _df_t = v.reset_index(drop=True)"); + code.appendLine(" _df_t.name = k"); + code.appendLine(" _df = pd.concat([_df, _df_t], axis=1)"); + code.append(" _df.columns = [[x[0] for x in _df.columns],[x[1] for x in _df.columns]]"); + + // Display - Statistics + if (statistics === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Statistics"); + code.appendLine("display(Markdown('### Statistics'))"); + code.appendLine("display(pd.DataFrame(data={'Count':_df.count(),'Mean':_df.mean(numeric_only=True),'Std. Deviation':_df.std(numeric_only=True),'Min':_df.min(),'Max':_df.max(),"); + code.appendLine(" 'Std. Error Mean':_df.apply(vp_sem),'Confidence interval':0.95,"); + code.append(" 'Lower':_df.apply(vp_confidence_interval).T[0],'Upper':_df.apply(vp_confidence_interval).T[1] }))"); + } + // Display - Boxplot + if (boxplot === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Boxplot"); + code.appendLine("import seaborn as sns"); + code.appendLine("import warnings"); + code.appendLine("with warnings.catch_warnings():"); + code.appendLine(" warnings.simplefilter(action='ignore', category=Warning)"); + code.appendLine(" sns.boxplot(data=_df)"); + code.append(" plt.show()"); + } + // Display - Equal Variance test + if (equalVariance === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Equal Variance test (Levene)"); + code.appendLine("from scipy import stats"); + code.appendLine("_lst = []"); + code.appendLine("_df.apply(lambda x: _lst.append(x.dropna()))"); + code.appendLine("_res = stats.levene(*_lst, center='mean')"); + code.appendLine("display(Markdown('### Equal Variance test (Levene)'))"); + code.append("display(pd.DataFrame(data={'Statistic':_res.statistic,'p-value':_res.pvalue}, index=['Equal Variance test (Levene)']))"); + } + + code.appendLine(); + code.appendLine(); + code.appendLine("# Two-way ANOVA"); + code.appendLine("import statsmodels.formula.api as smf"); + code.appendLine("from statsmodels.stats.anova import anova_lm"); + code.appendFormatLine("_model = smf.ols('{0} ~ C({1}) + C({2}) + C({3}):C({4})', vp_df)", depVarText, factorAText, factorBText, factorAText, factorBText); + code.appendLine("_result = _model.fit()"); + code.appendLine("_dfr = anova_lm(_result)"); + code.appendLine("_dfr.loc['Total','df'] = _dfr['df'].sum()"); + code.appendLine("_dfr.loc['Total','sum_sq'] = _dfr['sum_sq'].sum()"); + code.appendLine("display(Markdown('### Two-way ANOVA'))"); + code.append("display(_dfr)"); + + // Display - Interaction plot + if (interPlot === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Interaction plot"); + code.appendLine("from statsmodels.graphics.factorplots import interaction_plot"); + code.appendLine("import warnings"); + code.appendLine("with warnings.catch_warnings():"); + code.appendLine(" warnings.simplefilter(action='ignore', category=Warning)"); + code.appendFormatLine(" fig = interaction_plot(x=vp_df[{0}], trace=vp_df[{1}], response=vp_df[{2}])", factorA, factorB, depVar); + code.append(" plt.show()"); + } + // Post hoc analysis - Tukey HSD + if (tukeyHSD === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Post-hoc: Tukey HSD"); + code.appendLine("from statsmodels.sandbox.stats.multicomp import MultiComparison"); + code.appendFormatLine("_res = MultiComparison(vp_df[{0}], vp_df[{1}]).tukeyhsd(alpha={2})", depVar, factorA, sigLevel); + code.appendLine("display(Markdown('### Post-hoc: Tukey HSD'))"); + code.append("display(_res.summary())"); + } + // Post hoc analysis - Bonferroni + if (bonferroni === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Post-hoc: Bonferroni"); + code.appendLine("from statsmodels.sandbox.stats.multicomp import MultiComparison"); + code.appendFormatLine("_res = MultiComparison(vp_df[{0}], vp_df[{1}]).allpairtest(stats.ttest_ind,alpha={2},method='bonf')", depVar, factorA, sigLevel); + code.appendLine("display(Markdown('### Post-hoc: Bonferroni'))"); + code.append("display(_res[0])"); + } + if (tukey === true || scheffe === true || duncan === true) { + // Post hoc analysis - Tukey + if (tukey === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Post-hoc: Tukey"); + code.appendLine("import scikit_posthocs as sph"); + code.appendLine("display(Markdown('### Post-hoc: Tukey'))"); + code.appendFormat("display(sph.posthoc_tukey(vp_df, val_col={0}, group_col={1}))", depVar, factorA); + } + // Post hoc analysis - Scheffe + if (scheffe === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Post-hoc: Scheffe"); + code.appendLine("import scikit_posthocs as sph"); + code.appendLine("display(Markdown('### Post-hoc: Scheffe'))"); + code.appendFormat("display(sph.posthoc_scheffe(vp_df, val_col={0}, group_col={1}))", depVar, factorA); + } + // Post hoc analysis - Duncan + if (duncan === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Post-hoc: Duncan"); + code.appendLine("import scikit_posthocs as sph"); + code.appendLine("display(Markdown('### Post-hoc: Duncan'))"); + code.appendFormat("display(sph.posthoc_dunn(vp_df, val_col={0}, group_col={1}))", depVar, factorA); + } + } + break; + case 'ancova': + // 1. ANCOVA + code.appendLine(); + code.appendLine(); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("_df = pd.DataFrame()"); + code.appendFormatLine("for k, v in dict(list(vp_df.groupby({0})[{1}])).items():", factor, depVar); + code.appendLine(" _df_t = v.reset_index(drop=True)"); + code.appendLine(" _df_t.name = k"); + code.append(" _df = pd.concat([_df, _df_t], axis=1)"); + + // Display - Statistics + if (statistics === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Statistics"); + code.appendLine("display(Markdown('### Statistics'))"); + code.appendLine("display(pd.DataFrame(data={'Count':_df.count(),'Mean':_df.mean(numeric_only=True),'Std. Deviation':_df.std(numeric_only=True),'Min':_df.min(),'Max':_df.max(),"); + code.appendLine(" 'Std. Error Mean':_df.apply(vp_sem),'Confidence interval':0.95,"); + code.append(" 'Lower':_df.apply(vp_confidence_interval).T[0],'Upper':_df.apply(vp_confidence_interval).T[1] }))"); + } + // Display - Boxplot + if (boxplot === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Boxplot"); + code.appendLine("import seaborn as sns"); + code.appendLine("import warnings"); + code.appendLine("with warnings.catch_warnings():"); + code.appendLine(" warnings.simplefilter(action='ignore', category=Warning)"); + code.appendLine(" sns.boxplot(data=_df)"); + code.append(" plt.show()"); + } + + code.appendLine(); + code.appendLine(); + code.appendLine("# ANCOVA - Analysis of covariance"); + code.appendLine("import pingouin as pg"); + code.appendLine("display(Markdown('### ANCOVA - Analysis of covariance'))"); + code.appendFormat("display(pg.ancova(data=vp_df, dv={0}, between={1}, covar={2}))", depVar, factor, covariate); + break; + } + + codeList.push(code.toString()); + + return codeList; + } + + } + + return Anova; +}); \ No newline at end of file diff --git a/visualpython/js/m_stats/Chi2test.js b/visualpython/js/m_stats/Chi2test.js new file mode 100644 index 00000000..03d3cab2 --- /dev/null +++ b/visualpython/js/m_stats/Chi2test.js @@ -0,0 +1,215 @@ +/* + * Project Name : Visual Python + * Description : GUI-based Python code generator + * File Name : Chi2test.js + * Author : Black Logic + * Note : Chi-square test of independence + * License : GNU GPLv3 with Visual Python special exception + * Date : 2023. 05. 24 + * Change Date : + */ + +//============================================================================ +// [CLASS] Chi2test +//============================================================================ +define([ + __VP_TEXT_LOADER__('vp_base/html/m_stats/chi2test.html'), + 'vp_base/js/com/com_util', + 'vp_base/js/com/com_Const', + 'vp_base/js/com/com_String', + 'vp_base/js/com/com_generatorV2', + 'vp_base/js/com/component/PopupComponent', + 'vp_base/js/com/component/DataSelector', + 'vp_base/js/m_apps/Subset' +], function(nmHTML, com_util, com_Const, com_String, com_generator, PopupComponent, DataSelector, Subset) { + + /** + * Chi2test + */ + class Chi2test extends PopupComponent { + _init() { + super._init(); + /** Write codes executed before rendering */ + this.config.sizeLevel = 2; + this.config.checkModules = ['pd']; + this.config.docs = 'https://docs.scipy.org/doc/scipy/reference/'; + + this.state = { + data: '', + dataType: '', + row: '', + column: '', + barplot: true, + crossTab: true, + cramersCoef: true, + ...this.state + }; + + this.subsetEditor = null; + } + + _bindEvent() { + super._bindEvent(); + /** Implement binding events */ + var that = this; + + $(this.wrapSelector('#data')).on('change', function() { + let bindIdList = ['row', 'column']; + if (that.state.dataType === 'Series') { + // Series + bindIdList.forEach(id => { + $(that.wrapSelector('#' + id)).html(''); + $(that.wrapSelector('#' + id)).prop('disabled', true); + }); + } else { + // DataFrame + bindIdList.forEach(id => { + $(that.wrapSelector('#' + id)).prop('disabled', false); + }); + com_generator.vp_bindColumnSource(that, 'data', bindIdList, 'select', false, false); + } + }); + } + + handleVariableChange(data) { + this.state.data = data; + let bindIdList = ['row', 'column']; + if (that.state.dataType === 'DataFrame') { + // DataFrame + bindIdList.forEach(id => { + $(that.wrapSelector('#' + id)).html(''); + $(that.wrapSelector('#' + id)).prop('disabled', false); + }); + com_generator.vp_bindColumnSource(that, 'data', bindIdList, 'select', false, false); + } else { + // Others + bindIdList.forEach(id => { + $(that.wrapSelector('#' + id)).html(''); + $(that.wrapSelector('#' + id)).prop('disabled', true); + }); + } + } + + templateForBody() { + let page = $(nmHTML); + let that = this; + + let dataSelector = new DataSelector({ + pageThis: this, id: 'data', placeholder: 'Select data', required: true, boxClasses: 'vp-flex-gap5', + allowDataType: ['DataFrame'], withPopup: false, + finish: function(data, type) { + that.state.data = data; + that.state.dataType = type; + $(that.wrapSelector('#data')).trigger('change'); + }, + select: function(data, type) { + that.state.data = data; + that.state.dataType = type; + $(that.wrapSelector('#data')).trigger('change'); + } + }); + $(page).find('#data').replaceWith(dataSelector.toTagString()); + + return page; + } + + render() { + super.render(); + let that = this; + + // render Subset + this.subsetEditor = new Subset({ + pandasObject: '', + config: { name: 'Subset', category: this.name } }, + { + useAsModule: true, + useInputColumns: true, + targetSelector: this.wrapSelector('#data'), + pageThis: this, + finish: function(code, state) { + that.state.data = code; + that.state.dataType = state.returnType; + $(that.wrapSelector('#data')).val(code); + $(that.wrapSelector('#data')).trigger('change'); + } + }); + + // bind column if data exist + if (this.state.data !== '') { + com_generator.vp_bindColumnSource(this, 'data', ['row', 'column'], 'select', false, false); + } + } + + generateCode() { + let { data, row, column, barplot, crossTab, cramersCoef } = this.state; + let codeList = []; + let code = new com_String(); + + code.appendFormatLine("vp_df = {0}.dropna().copy()", data); + + // Display option + if (barplot === true) { + code.appendLine(); + code.appendLine("# Count plot"); + code.appendLine("import seaborn as sns"); + code.appendLine("import warnings"); + code.appendLine("with warnings.catch_warnings():"); + code.appendLine(" warnings.simplefilter(action='ignore', category=Warning)"); + code.appendFormatLine(" sns.countplot(data=vp_df, x={0}, hue={1})", row, column); + code.appendLine(" plt.show()"); + } + + code.appendLine(""); + code.appendLine("# Chi-square test of independence"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("from scipy import stats"); + code.appendFormatLine("_obs = pd.crosstab(index=vp_df[{0}], columns=vp_df[{1}])", row, column); + code.appendLine("_res1 = stats.chi2_contingency(_obs)"); + code.appendLine("_res2 = stats.chi2_contingency(_obs, lambda_='log-likelihood')"); + + if (crossTab === true) { + code.appendLine(""); + code.appendLine("# Cross tabulation: Count"); + code.appendFormatLine("_dfc = pd.crosstab(index=vp_df[{0}],columns=vp_df[{1}],margins=True,margins_name='Total')", row, column); + code.appendLine("_dfc = _dfc.reset_index().reset_index()"); + code.appendLine("_dfc[' '] = 'Count'"); + code.appendLine(""); + code.appendLine("# Cross tabulation: Expected count"); + code.appendLine("_dfe = pd.DataFrame(_res1.expected_freq, index=_obs.index, columns=_obs.columns).round(1)"); + code.appendLine("_dfe.loc['Total',:] = _dfe.sum(axis=0)"); + code.appendLine("_dfe.loc[:,'Total'] = _dfe.sum(axis=1)"); + code.appendLine("_dfe = _dfe.reset_index().reset_index()"); + code.appendLine("_dfe[' '] = 'Expected count'"); + code.appendLine(""); + code.appendLine("# Cross tabulation: Count + Expected count"); + code.appendLine("display(Markdown('### Cross tabulation'))"); + code.appendFormatLine("display(pd.concat([_dfc, _dfe]).set_index([{0},' ']).sort_values('index').drop('index',axis=1))", row); + } + + code.appendLine(""); + code.appendLine("# Chi-square test"); + code.appendLine("display(Markdown('### Chi-square test'))"); + code.appendLine("display(pd.DataFrame(data = {'Value':[_res1.statistic,_res2.statistic,vp_df.dropna().shape[0]],"); + code.appendLine(" 'df':[_res1.dof,_res2.dof,np.nan],"); + code.appendLine(" 'p-value(two-sided)':[_res1.pvalue,_res2.pvalue,np.nan]},"); + code.append(" index= ['Pearson Chi-square','Likelihood ratio','N of valid cases']))"); + + if (cramersCoef === true) { + code.appendLine(""); + code.appendLine(""); + code.appendLine("# Cramers' V coefficient"); + code.appendLine("_X2 = stats.chi2_contingency(_obs)[0]"); + code.appendLine("_sum = _obs.sum().sum()"); + code.appendLine("_minDim = min(_obs.shape)-1"); + code.appendLine("display(Markdown('### Cramers V coefficient'))"); + code.append("display(pd.DataFrame(data={'Value':np.sqrt((_X2/_sum) / _minDim)}, index=['Cramers V coefficient']))"); + } + codeList.push(code.toString()); + + return codeList; + } + + } + + return Chi2test; +}); \ No newline at end of file diff --git a/visualpython/js/m_stats/CorrAnalysis.js b/visualpython/js/m_stats/CorrAnalysis.js new file mode 100644 index 00000000..92251434 --- /dev/null +++ b/visualpython/js/m_stats/CorrAnalysis.js @@ -0,0 +1,202 @@ +/* + * Project Name : Visual Python + * Description : GUI-based Python code generator + * File Name : CorrAnalysis.js + * Author : Black Logic + * Note : Correlation Analysis + * License : GNU GPLv3 with Visual Python special exception + * Date : 2023. 05. 24 + * Change Date : + */ + +//============================================================================ +// [CLASS] CorrAnalysis +//============================================================================ +define([ + __VP_TEXT_LOADER__('vp_base/html/m_stats/corrAnalysis.html'), + 'vp_base/js/com/com_util', + 'vp_base/js/com/com_Const', + 'vp_base/js/com/com_String', + 'vp_base/js/com/component/PopupComponent', + 'vp_base/js/com/component/DataSelector', + 'vp_base/js/com/component/MultiSelector', + 'vp_base/js/m_apps/Subset' +], function(eqHTML, com_util, com_Const, com_String, PopupComponent, DataSelector, MultiSelector, Subset) { + + /** + * CorrAnalysis + */ + class CorrAnalysis extends PopupComponent { + _init() { + super._init(); + /** Write codes executed before rendering */ + this.config.sizeLevel = 2; + this.config.checkModules = ['pd']; + this.config.docs = 'https://docs.scipy.org/doc/scipy/reference/'; + + this.state = { + data: '', + variable: [], + corrType: 'pearson', + corrAnlaysis: true, + corrMatrix: true, + corrHeatmap: false, + scatterMatrix: false, + ...this.state + }; + + this.subsetEditor = null; + this.columnSelector = null; + } + + _bindEvent() { + super._bindEvent(); + /** Implement binding events */ + var that = this; + + $(this.wrapSelector('#data')).on('change', function() { + let data = $(this).val(); + that.handleVariableChange(data); + }); + } + + handleVariableChange(data) { + this.state.data = data; + // render variable selector + this.columnSelector = new MultiSelector(this.wrapSelector('#variable'), + { mode: 'columns', parent: data, showDescription: false } + ); + } + + templateForBody() { + let page = $(eqHTML); + let that = this; + + // generate dataselector + let dataSelector = new DataSelector({ + pageThis: this, id: 'data', placeholder: 'Select data', required: true, boxClasses: 'vp-flex-gap5', + allowDataType: ['DataFrame'], withPopup: false, + finish: function(data, type) { + that.state.data = data; + $(that.wrapSelector('#data')).trigger('change'); + }, + select: function(data, type) { + that.state.data = data; + $(that.wrapSelector('#data')).trigger('change'); + } + }); + $(page).find('#data').replaceWith(dataSelector.toTagString()); + + return page; + } + + render() { + super.render(); + let that = this; + + // render Subset + this.subsetEditor = new Subset({ + pandasObject: '', + config: { name: 'Subset', category: this.name } }, + { + useAsModule: true, + useInputColumns: true, + targetSelector: this.wrapSelector('#data'), + pageThis: this, + finish: function(code) { + $(that.wrapSelector('#data')).val(code); + that.handleVariableChange(code); + } + }); + + // render variable selector + this.columnSelector = new MultiSelector(this.wrapSelector('#variable'), + { mode: 'columns', parent: this.state.data, selectedList: this.state.variable?.map(x=>x.code), showDescription: false } + ); + } + + generateCode() { + let { data, variable, corrType, corrAnlaysis, corrMatrix, corrHeatmap, scatterMatrix } = this.state; + let codeList = []; + let code = new com_String(); + + // data declaration + code.appendFormat("vp_df = {0}", data); + if (this.columnSelector) { + let columns = this.columnSelector.getDataList(); + this.state.variable = columns; + if (columns.length > 0) { + code.appendFormat("[[{0}]]", columns.map(x => x.code).join(', ')); + } + } + code.append('.dropna().copy()'); + + let corrTypeLabel = $(this.wrapSelector('#corrType option:selected')).text(); + + // Display option : Correlation Analysis + if (corrAnlaysis === true) { + // Inner function : vp_confidence_interval_corr + this.addCheckModules('vp_confidence_interval_corr'); + + code.appendLine(); + code.appendLine(); + code.appendLine("# Correlation Analysis"); + code.appendLine("from scipy import stats"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("_dfr = pd.DataFrame()"); + code.appendLine("for i, col1 in enumerate(vp_df.columns):"); + code.appendLine(" for j, col2 in enumerate(vp_df.columns):"); + code.appendLine(" if i >= j: continue"); + code.appendLine(" if pd.api.types.is_numeric_dtype(vp_df[col1]) and pd.api.types.is_numeric_dtype(vp_df[col2]):"); + code.appendFormatLine(" _res = vp_confidence_interval_corr(vp_df[col1], vp_df[col2], method='{0}')", corrType); + code.appendLine(" _df_t = pd.DataFrame(data={'Variable1':col1,'Variable2':col2,'N':vp_df[col1].size,'Correlation coefficient':_res[0],"); + code.appendLine(" 'p-value':_res[1],'Lower(95%)':_res[2],'Upper(95%)':_res[3]}, index=[0])"); + code.appendLine(" _dfr = pd.concat([_dfr, _df_t]).reset_index(drop=True)"); + code.appendFormatLine("display(Markdown('### Correlation Analysis: {0}'))", corrTypeLabel.replace("'", "\\'")); + code.append("display(_dfr)"); + } + + // Display option : Correlation Matrix + if (corrMatrix === true) { + code.appendLine(); + code.appendLine(); + code.appendFormatLine("# Correlation matrix: {0}", corrTypeLabel); + code.appendLine("from IPython.display import display"); + code.appendFormat("display(vp_df.corr(method='{0}', numeric_only=True).round(2))", corrType); + } + + if (corrHeatmap === true || scatterMatrix === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Chart"); + code.appendLine("import seaborn as sns"); + code.appendLine("import warnings"); + code.appendLine("with warnings.catch_warnings():"); + code.append(" warnings.simplefilter(action='ignore', category=Warning)"); + // Display option : Correlation Heatmap + if (corrHeatmap === true) { + code.appendLine(); + code.appendLine(); + code.appendLine(" # Heatmap"); + code.appendFormatLine(" sns.heatmap(vp_df.corr(method='{0}', numeric_only=True), annot=True, fmt='.2f', cmap='coolwarm')", corrType); + code.appendFormatLine(" plt.title('Correlation heatmap: {0}')", corrTypeLabel.replace("'", "\\'")); + code.append(" plt.show()"); + } + // Display option : Scatter Matrix + if (scatterMatrix === true) { + code.appendLine(); + code.appendLine(); + code.appendLine(" # Scatter matrix"); + code.appendLine(" pd.plotting.scatter_matrix(vp_df)"); + code.append(" plt.show()"); + } + } + codeList.push(code.toString()); + + return codeList; + } + + } + + return CorrAnalysis; +}); \ No newline at end of file diff --git a/visualpython/js/m_stats/DescStats.js b/visualpython/js/m_stats/DescStats.js new file mode 100644 index 00000000..e6a9aa45 --- /dev/null +++ b/visualpython/js/m_stats/DescStats.js @@ -0,0 +1,320 @@ +/* + * Project Name : Visual Python + * Description : GUI-based Python code generator + * File Name : DescStats.js + * Author : Black Logic + * Note : Descriptive Statistics + * License : GNU GPLv3 with Visual Python special exception + * Date : 2023. 05. 31 + * Change Date : + */ + +//============================================================================ +// [CLASS] DescStats +//============================================================================ +define([ + __VP_TEXT_LOADER__('vp_base/html/m_stats/descStats.html'), + __VP_CSS_LOADER__('vp_base/css/m_stats/descStats'), + 'vp_base/js/com/com_util', + 'vp_base/js/com/com_Const', + 'vp_base/js/com/com_String', + 'vp_base/js/com/component/PopupComponent', + 'vp_base/js/com/component/DataSelector', + 'vp_base/js/com/component/MultiSelector', + 'vp_base/js/m_apps/Subset' +], function(eqHTML, dsCss, com_util, com_Const, com_String, PopupComponent, DataSelector, MultiSelector, Subset) { + + /** + * DescStats + */ + class DescStats extends PopupComponent { + _init() { + super._init(); + /** Write codes executed before rendering */ + this.config.sizeLevel = 2; + this.config.checkModules = ['pd']; + + this.state = { + data: '', + variable: [], + // Central tendency + mean: true, + median: false, + mode: false, + sum: true, + // Dispersion + min: false, + max: false, + range: false, + std: true, + var: true, + semean: false, + skew: false, + kurtosis: false, + // Percentile values + quantile: true, + usePercentile: false, + percentiles: [], + // Frequency table + frequency: true, + percent: true, + validPercent: true, + cumulativePercent: true, + noUniqVals: 10, + // Display + histogram: true, + scatterMatrix: true, + boxplot: true, + ...this.state + }; + + this.subsetEditor = null; + this.columnSelector = null; + } + + _bindEvent() { + super._bindEvent(); + /** Implement binding events */ + var that = this; + + // data selection + $(this.wrapSelector('#data')).on('change', function() { + let data = $(this).val(); + that.handleVariableChange(data); + }); + + // use percentile + $(this.wrapSelector('#usePercentile')).on('change', function() { + let checked = $(this).prop('checked'); + if (checked === true) { + // enable percentile editing + $(that.wrapSelector('#percentile')).prop('disabled', false); + $(that.wrapSelector('#addPercentile')).prop('disabled', false); + $(that.wrapSelector('.vp-percentile-box')).removeClass('disabled'); + } else { + // disable percentile editing + $(that.wrapSelector('#percentile')).prop('disabled', true); + $(that.wrapSelector('#addPercentile')).prop('disabled', true); + $(that.wrapSelector('.vp-percentile-box')).addClass('disabled'); + + } + }); + + // add percentile + $(this.wrapSelector('#addPercentile')).on('click', function() { + let newVal = $(that.wrapSelector('#percentile')).val(); + if (newVal && newVal !== '') { + let newValNum = parseInt(newVal); + that.addPercentile(newValNum); + that.state.percentiles.push(newValNum); + $(that.wrapSelector('#percentile')).val(''); + } + }); + } + + handleVariableChange(data) { + this.state.data = data; + // render variable selector + this.columnSelector = new MultiSelector(this.wrapSelector('#variable'), + { mode: 'columns', parent: data, showDescription: false } + ); + } + + addPercentile(percentile) { + if (this.state.percentiles.indexOf(percentile) === -1) { + $(this.wrapSelector('.vp-percentile-box')).append( + $(`

${percentile}

`)); + + // delete percentile + let that = this; + $(this.wrapSelector('.vp-percentile-box:not(.disabled) .vp-percentile-remove')).on('click', function() { + if (that.state.usePercentile === true) { + let delVal = parseInt($(this).parent().find('.vp-percentile-value').text()); + that.state.percentiles = that.state.percentiles.filter(x => x !== delVal); + $(this).closest('.vp-percentile-item').remove(); + } + }); + } + } + + templateForBody() { + let page = $(eqHTML); + let that = this; + + // generate dataselector + let dataSelector = new DataSelector({ + pageThis: this, id: 'data', placeholder: 'Select data', required: true, boxClasses: 'vp-flex-gap5', + allowDataType: ['DataFrame'], withPopup: false, + finish: function(data, type) { + that.state.data = data; + $(that.wrapSelector('#data')).trigger('change'); + }, + select: function(data, type) { + that.state.data = data; + $(that.wrapSelector('#data')).trigger('change'); + } + }); + $(page).find('#data').replaceWith(dataSelector.toTagString()); + + return page; + } + + render() { + super.render(); + let that = this; + + // render Subset + this.subsetEditor = new Subset({ + pandasObject: '', + config: { name: 'Subset', category: this.name } }, + { + useAsModule: true, + useInputColumns: true, + targetSelector: this.wrapSelector('#data'), + pageThis: this, + finish: function(code) { + $(that.wrapSelector('#data')).val(code); + that.handleVariableChange(code); + } + }); + + // render variable selector + this.columnSelector = new MultiSelector(this.wrapSelector('#variable'), + { mode: 'columns', parent: this.state.data, showDescription: false } + ); + } + + generateCode() { + let { data, variable, + // Central tendency + mean,median,mode,sum, + // Dispersion + min,max,range,std,variance,semean,skew,kurtosis, + // Percentile values + quantile,usePercentile,percentiles, + // Frequency table + frequency,percent,validPercent,cumulativePercent,noUniqVals, + // Display + histogram,scatterMatrix,boxplot + } = this.state; + let codeList = []; + let code = new com_String(); + + // data declaration + code.appendFormat("vp_df = {0}", data); + if (this.columnSelector) { + let columns = this.columnSelector.getDataList(); + if (columns.length > 0) { + code.appendFormat("[[{0}]]", columns.map(x => x.code).join(', ')); + } + } + code.appendLine('.copy()'); + + // Descriptive statistics + code.appendLine(); + code.appendLine("# Descriptive statistics"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("display(Markdown('### Descriptive statistics'))"); + code.appendLine("display(pd.DataFrame({"); + code.appendLine(" 'N Total':vp_df.shape[0],"); + code.appendLine(" 'N Valid':vp_df.count(numeric_only=True),"); + code.appendLine(" 'N Missing':vp_df.loc[:,vp_df.apply(pd.api.types.is_numeric_dtype)].isnull().sum(),"); + if (mean === true) code.appendLine(" 'Mean':vp_df.mean(numeric_only=True),"); + if (median === true) code.appendLine(" 'Median':vp_df.median(numeric_only=True),"); + if (mode === true) code.appendLine(" 'Mode':vp_df.mode(numeric_only=True).iloc[0],"); + if (sum === true) code.appendLine(" 'Sum':vp_df.sum(numeric_only=True),"); + if (min === true) code.appendLine(" 'Minimun':vp_df.min(numeric_only=True),"); + if (max === true) code.appendLine(" 'Maximum':vp_df.max(numeric_only=True),"); + if (range === true) code.appendLine(" 'Range':vp_df.max(numeric_only=True) - vp_df.min(numeric_only=True),"); + if (std === true) code.appendLine(" 'Std. deviation':vp_df.std(numeric_only=True),"); + if (variance === true) code.appendLine(" 'Variance':vp_df.var(numeric_only=True),"); + if (semean === true) code.appendLine(" 'S.E. mean':vp_df.std(numeric_only=True)/np.sqrt(vp_df.count(numeric_only=True)),"); + if (skew === true) code.appendLine(" 'Skewness':vp_df.skew(numeric_only=True),"); + if (kurtosis === true) code.appendLine(" 'Kurtosis':vp_df.kurtosis(numeric_only=True),"); + let sortedPercentiles = []; + if (quantile === true) { + sortedPercentiles = [25, 50, 75]; + } + if (usePercentile === true && percentiles.length > 0) { + sortedPercentiles = [...sortedPercentiles, ...percentiles]; + } + sortedPercentiles.sort((a, b) => { return a - b; }); + sortedPercentiles.forEach(num => { + code.appendFormatLine(" 'Percentile: {0}':vp_df.quantile(q={1}, numeric_only=True),", num, (num / 100).toFixed(2)); + }); + code.appendLine("}).round(3).T)"); + + // Frequency table + code.appendLine(); + code.appendLine("# Frequency table"); + code.appendLine("display(Markdown('### Frequency table'))"); + code.appendLine("for col in vp_df.columns:"); + code.appendFormatLine(" if pd.api.types.is_numeric_dtype(vp_df[col]) and vp_df[col].value_counts().size > {0}:", noUniqVals); + code.appendFormatLine(" _bins = {0}", noUniqVals); + code.appendLine(" else: _bins = None"); + code.appendLine(" "); + code.appendLine(" _dfr = pd.DataFrame({"); + if (frequency === true) code.appendLine(" 'Frequency':vp_df[col].value_counts(bins=_bins, sort=False),"); + if (percent === true) code.appendLine(" 'Percent':100*(vp_df[col].value_counts(bins=_bins, sort=False) / vp_df[col].size),"); + if (validPercent === true) code.appendLine(" 'Valid percent':100*(vp_df[col].value_counts(bins=_bins, sort=False)/vp_df[col].count())"); + code.appendLine("}).round(2)"); + if (cumulativePercent === true) code.appendLine(" _dfr['Cumulative percent'] = _dfr['Valid percent'].cumsum()"); + code.appendLine(" _dfr.loc['N Valid',:] = _dfr.iloc[:,:3].sum()"); + code.appendLine(" _dfr.loc['N Missing','Frequency'] = vp_df[col].isnull().sum()"); + code.appendLine(" _dfr.loc['N Total','Frequency'] = vp_df[col].size"); + code.append(" display(_dfr)"); + + // Display option + if (histogram || scatterMatrix || boxplot) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Charts"); + code.appendLine("import seaborn as sns"); + code.appendLine("import warnings"); + code.appendLine("with warnings.catch_warnings():"); + code.append(" warnings.simplefilter(action='ignore', category=Warning)"); + if (histogram === true) { + code.appendLine(); + code.appendLine(); + code.appendLine(" # Histogram"); + code.appendLine(" idx = 1"); + code.appendLine(" for col in vp_df.columns:"); + code.appendLine(" plt.subplot(2,2, idx)"); + code.appendFormatLine(" if pd.api.types.is_numeric_dtype(vp_df[col]) and vp_df[col].value_counts().size > {0}:", noUniqVals); + code.appendLine(" sns.histplot(data=vp_df, x=col, kde=True)"); + code.appendLine(" else:"); + code.appendLine(" sns.countplot(data=vp_df, x=col)"); + code.appendLine(" "); + code.appendLine(" if idx < 4:"); + code.appendLine(" idx += 1"); + code.appendLine(" else:"); + code.appendLine(" idx = 1"); + code.appendLine(" plt.tight_layout()"); + code.append(" plt.show()"); + } + if (scatterMatrix === true) { + code.appendLine(); + code.appendLine(); + code.appendLine(" # Scatter matrix "); + code.appendLine(" pd.plotting.scatter_matrix(vp_df, marker='o', hist_kwds={'bins': 30}, s=30, alpha=.8)"); + code.append(" plt.show()"); + } + if (boxplot === true) { + code.appendLine(); + code.appendLine(); + code.appendLine(" # Boxplot"); + code.appendLine(" sns.boxplot(vp_df)"); + code.append(" plt.show()"); + } + } + + return code.toString(); + } + + } + + return DescStats; +}); \ No newline at end of file diff --git a/visualpython/js/m_stats/EqualVarTest.js b/visualpython/js/m_stats/EqualVarTest.js new file mode 100644 index 00000000..3f6569e1 --- /dev/null +++ b/visualpython/js/m_stats/EqualVarTest.js @@ -0,0 +1,246 @@ +/* + * Project Name : Visual Python + * Description : GUI-based Python code generator + * File Name : EqualVarTest.js + * Author : Black Logic + * Note : Equal Variance test + * License : GNU GPLv3 with Visual Python special exception + * Date : 2023. 05. 09 + * Change Date : + */ + +//============================================================================ +// [CLASS] EqualVarTest +//============================================================================ +define([ + __VP_TEXT_LOADER__('vp_base/html/m_stats/equalVarTest.html'), + 'vp_base/js/com/com_util', + 'vp_base/js/com/com_Const', + 'vp_base/js/com/com_String', + 'vp_base/js/com/com_generatorV2', + 'vp_base/js/com/component/PopupComponent', + 'vp_base/js/com/component/DataSelector', + 'vp_base/js/com/component/MultiSelector', + 'vp_base/js/m_apps/Subset' +], function(eqHTML, com_util, com_Const, com_String, com_generator, PopupComponent, DataSelector, MultiSelector, Subset) { + + /** + * EqualVarTest + */ + class EqualVarTest extends PopupComponent { + _init() { + super._init(); + /** Write codes executed before rendering */ + this.config.sizeLevel = 2; + this.config.checkModules = ['pd']; + this.config.docs = 'https://docs.scipy.org/doc/scipy/reference/'; + + this.state = { + testType: 'bartlett', + inputType: 'long-data', + data: '', + variableMulti: [], + variable: '', + factor: '', + center: 'mean', + histogram: true, + ...this.state + }; + + this.subsetEditor = {}; + this.columnSelector = {}; + } + + _bindEvent() { + super._bindEvent(); + /** Implement binding events */ + var that = this; + + // change test type + $(this.wrapSelector('#testType')).on('change', function() { + let testType = $(this).val(); + that.state.testType = testType; + + $(that.wrapSelector('.vp-st-option')).hide(); + $(that.wrapSelector('.vp-st-option.' + testType)).show(); + }); + + // change input type + $(this.wrapSelector('input[name="inputType"]:radio')).on('change', function() { + let inputType = $(this).val(); + that.state.inputType = inputType; + $(that.wrapSelector('.vp-variable-box')).hide(); + $(that.wrapSelector('.vp-variable-box.' + inputType)).show(); + }); + + // data change event + $(this.wrapSelector('#data')).on('change', function() { + let data = $(this).val(); + that.handleVariableChange(data); + }); + } + + handleVariableChange(data) { + this.state.data = data; + // render column selector + com_generator.vp_bindColumnSource(this, 'data', ['variable', 'factor'], 'select', false, false); + // render variable selector + this.columnSelector = new MultiSelector(this.wrapSelector('#variableMulti'), + { mode: 'columns', parent: data, showDescription: false } + ); + } + + templateForBody() { + let page = $(eqHTML); + let that = this; + + // generate dataselector + let dataSelector = new DataSelector({ + pageThis: this, id: 'data', placeholder: 'Select data', required: true, boxClasses: 'vp-flex-gap5', + allowDataType: ['DataFrame'], withPopup: false, + finish: function(data, type) { + that.state.data = data; + $(that.wrapSelector('#data')).trigger('change'); + }, + select: function(data, type) { + that.state.data = data; + $(that.wrapSelector('#data')).trigger('change'); + } + }); + $(page).find('#data').replaceWith(dataSelector.toTagString()); + + return page; + } + + render() { + super.render(); + let that = this; + + // render Subset + this.subsetEditor = new Subset({ + pandasObject: '', + config: { name: 'Subset', category: this.name } }, + { + useAsModule: true, + useInputColumns: true, + targetSelector: this.wrapSelector('#data'), + pageThis: this, + finish: function(code) { + $(that.wrapSelector('#data')).val(code); + that.handleVariableChange(code); + } + }); + + if (this.state.data !== '') { + // render column selector + com_generator.vp_bindColumnSource(this, 'data', ['variable', 'factor'], 'select', false, false); + } + // render variable selector + this.columnSelector = new MultiSelector(this.wrapSelector('#variableMulti'), + { mode: 'columns', parent: this.state.data, selectedList: this.state.variableMulti?.map(x => x.code), showDescription: false } + ); + + $(this.wrapSelector('.vp-variable-box')).hide(); + $(this.wrapSelector('.vp-variable-box.' + this.state.inputType)).show(); + + // control display option + $(this.wrapSelector('.vp-st-option')).hide(); + $(this.wrapSelector('.vp-st-option.' + this.state.testType)).show(); + } + + generateCode() { + let { testType, inputType, data, variable, factor, center, histogram } = this.state; + let codeList = []; + let code = new com_String(); + let that = this; + + // test type label + let testTypeLabel = $(this.wrapSelector('#testType option:selected')).text(); + code.appendFormatLine("# {0}", testTypeLabel); + + if (inputType === 'long-data') { + code.appendFormatLine("vp_df = {0}.dropna().copy()", data); + code.appendLine("_df = pd.DataFrame()"); + code.appendFormatLine("for k, v in dict(list(vp_df.groupby({0})[{1}])).items():", factor, variable); + code.appendLine(" _df_t = v.reset_index(drop=True)"); + code.appendLine(" _df_t.name = k"); + code.append(" _df = pd.concat([_df, _df_t], axis=1)"); + } else if (inputType === 'wide-data') { + // get variable multi + let columns = this.columnSelector.getDataList(); + this.state.variableMulti = columns; + code.appendFormatLine("vp_df = {0}[[{1}]].copy()", data, columns.map(x => x.code).join(', ')); // without dropna + code.append("_df = vp_df.copy()"); + } + + // add variance code + code.appendLine(); + code.appendLine(); + code.appendLine("# Variance"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("from scipy import stats"); + code.appendLine("_dfr = _df.var().to_frame()"); + code.appendLine("_dfr.columns = ['Variance']"); + code.append("display(_dfr)"); + + switch (testType) { + case 'bartlett': + // 1. Bartlett test + code.appendLine(); + code.appendLine(); + code.appendLine("# Bartlett test"); + code.appendLine("_lst = []"); + code.appendLine("_df.apply(lambda x: _lst.append(x.dropna()))"); + code.appendLine("_res = stats.bartlett(*_lst)"); + code.appendLine("display(Markdown('### Bartlett test'))"); + code.appendLine("display(pd.DataFrame(data={'Statistic':_res.statistic,'p-value':_res.pvalue},"); + code.append(" index=['Equal Variance test (Bartlett)']))"); + break; + case 'levene': + // 1. Levene test + code.appendLine(); + code.appendLine(); + code.appendLine("# Levene test"); + code.appendLine("_lst = []"); + code.appendLine("_df.apply(lambda x: _lst.append(x.dropna()))"); + code.appendFormatLine("_res = stats.levene(*_lst, center='{0}')", center); + code.appendLine("display(Markdown('### Levene test'))"); + code.appendLine("display(pd.DataFrame(data={'Statistic':_res.statistic,'p-value':_res.pvalue},"); + code.append(" index=['Equal Variance test (Levene)']))"); + break; + case 'fligner': + // 1. Fligner test + code.appendLine(); + code.appendLine(); + code.appendLine("# Fligner test"); + code.appendLine("_lst = []"); + code.appendLine("_df.apply(lambda x: _lst.append(x.dropna()))"); + code.appendFormatLine("_res = stats.fligner(*_lst, center='{0}')", center); + code.appendLine("display(Markdown('### Fligner test'))"); + code.appendLine("display(pd.DataFrame(data={'Statistic':_res.statistic,'p-value':_res.pvalue},"); + code.append(" index=['Equal Variance test (Fligner)']))"); + break; + } + + // Display option + if (histogram === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Histogram"); + code.appendLine("import seaborn as sns"); + code.appendLine("import warnings"); + code.appendLine("with warnings.catch_warnings():"); + code.appendLine(" warnings.simplefilter(action='ignore', category=Warning)"); + code.appendLine(" sns.histplot(_df, stat='density', kde=True)"); + code.appendLine(" plt.title('Histogram')"); + code.append(" plt.show()"); + } + codeList.push(code.toString()); + + return codeList; + } + + } + + return EqualVarTest; +}); \ No newline at end of file diff --git a/visualpython/js/m_stats/FactorAnalysis.js b/visualpython/js/m_stats/FactorAnalysis.js new file mode 100644 index 00000000..188efe19 --- /dev/null +++ b/visualpython/js/m_stats/FactorAnalysis.js @@ -0,0 +1,301 @@ +/* + * Project Name : Visual Python + * Description : GUI-based Python code generator + * File Name : FactorAnalysis.js + * Author : Black Logic + * Note : Factor Analysis + * License : GNU GPLv3 with Visual Python special exception + * Date : 2023. 05. 24 + * Change Date : + */ + +//============================================================================ +// [CLASS] FactorAnalysis +//============================================================================ +define([ + __VP_TEXT_LOADER__('vp_base/html/m_stats/factorAnalysis.html'), + 'vp_base/js/com/com_util', + 'vp_base/js/com/com_Const', + 'vp_base/js/com/com_String', + 'vp_base/js/com/component/PopupComponent', + 'vp_base/js/com/component/DataSelector', + 'vp_base/js/com/component/MultiSelector', + 'vp_base/js/m_apps/Subset' +], function(eqHTML, com_util, com_Const, com_String, PopupComponent, DataSelector, MultiSelector, Subset) { + + /** + * FactorAnalysis + */ + class FactorAnalysis extends PopupComponent { + _init() { + super._init(); + /** Write codes executed before rendering */ + this.config.sizeLevel = 2; + this.config.checkModules = ['pd']; + this.config.installButton = true; + this.config.docs = 'https://factor-analyzer.readthedocs.io/en/latest/factor_analyzer.html'; + + this.state = { + data: '', + variable: [], + rotation: "'varimax'", + method: 'principal', + impute: 'drop', + extract: 'eigenvalue', + eigenvalue: 1, + factor: '', + corrMatrix: true, + screePlot: true, + ...this.state + }; + + this.rotationList = [ + { label: "None", value: "None" }, + { label: "varimax", value: "'varimax'" }, + { label: "promax", value: "'promax'" }, + { label: "oblimin", value: "'oblimin'" }, + { label: "oblimax", value: "'oblimax'" }, + { label: "quartimin", value: "'quartimin'" }, + { label: "quartimax", value: "'quartimax'" }, + { label: "equamax", value: "'equamax'" }, + ]; + this.methodList = [ + { label: "minres", value: "minres" }, + { label: "ml", value: "ml" }, + { label: "principal", value: "principal" }, + ]; + this.imputeList = [ + { label: "drop", value: "drop" }, + { label: "mean", value: "mean" }, + { label: "median", value: "median" }, + ] + + this.subsetEditor = null; + this.columnSelector = null; + } + + _bindEvent() { + super._bindEvent(); + /** Implement binding events */ + var that = this; + + $(this.wrapSelector('#data')).on('change', function() { + let data = $(this).val(); + that.handleVariableChange(data); + }); + } + + handleVariableChange(data) { + this.state.data = data; + this.state.variable = []; + // render variable selector + this.columnSelector = new MultiSelector(this.wrapSelector('#variable'), + { mode: 'columns', parent: data, showDescription: false } + ); + } + + templateForBody() { + let page = $(eqHTML); + let that = this; + + // generate dataselector + let dataSelector = new DataSelector({ + pageThis: this, id: 'data', placeholder: 'Select data', required: true, boxClasses: 'vp-flex-gap5', + allowDataType: ['DataFrame'], withPopup: false, + finish: function(data, type) { + that.state.data = data; + $(that.wrapSelector('#data')).trigger('change'); + }, + select: function(data, type) { + that.state.data = data; + $(that.wrapSelector('#data')).trigger('change'); + } + }); + $(page).find('#data').replaceWith(dataSelector.toTagString()); + + // generate rotation options + this.rotationList.forEach(obj => { + let selected = obj.value === that.state.rotation; + $(page).find('#rotation').append(``); + }); + + // generate method options + this.methodList.forEach(obj => { + let selected = obj.value === that.state.method; + $(page).find('#method').append(``); + }); + + // generate impute options + this.imputeList.forEach(obj => { + let selected = obj.value === that.state.impute; + $(page).find('#impute').append(``); + }); + + return page; + } + + render() { + super.render(); + let that = this; + + // render Subset + this.subsetEditor = new Subset({ + pandasObject: '', + config: { name: 'Subset', category: this.name } }, + { + useAsModule: true, + useInputColumns: true, + targetSelector: this.wrapSelector('#data'), + pageThis: this, + finish: function(code) { + $(that.wrapSelector('#data')).val(code); + that.handleVariableChange(code); + } + }); + + // render variable selector + this.columnSelector = new MultiSelector(this.wrapSelector('#variable'), + { mode: 'columns', parent: this.state.data, selectedList: this.state.variable.map(x=>x.code), showDescription: false } + ); + } + + generateInstallCode() { + let installCode = '!pip install factor-analyzer'; + // Add installation code + if (vpConfig.extensionType === 'lite') { + installCode = '%pip install factor-analyzer'; + } + return [ installCode ]; + } + + generateCode() { + let { data, variable, rotation, method, impute, extract, eigenvalue, factor, corrMatrix, screePlot } = this.state; + let codeList = []; + let code = new com_String(); + + // data declaration + code.appendFormat("vp_df = {0}", data); + if (this.columnSelector) { + let columns = this.columnSelector.getDataList(); + this.state.variable = columns; + if (columns.length > 0) { + code.appendFormat("[[{0}]]", columns.map(x => x.code).join(', ')); + } + } + code.appendLine('.dropna().copy()'); + + // KMO(Kaiser-Meyer-Olkin) measure of sampling adequacy + code.appendLine(); + code.appendLine("# KMO(Kaiser-Meyer-Olkin) measure of sampling adequacy"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("from factor_analyzer.factor_analyzer import calculate_kmo"); + code.appendLine("_kmo = calculate_kmo(vp_df)"); + code.appendLine("display(Markdown('### KMO measure of sampling adequacy'))"); + code.appendLine("display(pd.DataFrame(data={'Statistic ':_kmo[1]}, index=['KMO measure of sampling adequacy']))"); + + // Bartlett's test of sphericity + code.appendLine(); + code.appendLine("# Bartlett's test of sphericity"); + code.appendLine("from factor_analyzer.factor_analyzer import calculate_bartlett_sphericity"); + code.appendLine("_bartlett = calculate_bartlett_sphericity(vp_df)"); + code.appendLine("display(Markdown('### Bartlett\\'s test of sphericity'))"); + code.appendLine("display(pd.DataFrame(data={'Chi-square statistic':_bartlett[0],'p-value':_bartlett[1]}, index=['Bartlett test of sphericity']))"); + + // Initial of Factor Analysis + code.appendLine(); + code.appendLine("# Initial"); + code.appendLine("from factor_analyzer import FactorAnalyzer"); + code.appendFormatLine("_fa1 = FactorAnalyzer(n_factors=vp_df.shape[1], rotation=None, method='{0}', impute='{1}')", method, impute); + code.appendLine("_fa1.fit(vp_df)"); + + // Number of Factor + code.appendLine(); + code.appendLine("# Number of Factor"); + if (extract === 'eigenvalue') { + code.appendFormatLine("_nof = (_fa1.get_eigenvalues()[0] > {0}).sum()", eigenvalue); + } else if (extract === 'factor') { + code.appendFormatLine("_nof = {0}", factor); + } + + // Unrotated + code.appendLine(); + code.appendLine("# Un-rotated"); + code.appendFormatLine("_fa2 = FactorAnalyzer(n_factors=_nof, rotation=None, method='{0}', impute='{1}')", method, impute); + code.appendLine("_fa2.fit(vp_df)"); + + // Rotated + code.appendLine(); + code.appendLine("# Rotated"); + code.appendFormatLine("_fa3 = FactorAnalyzer(n_factors=_nof, rotation={0}, method='{1}', impute='{2}')", rotation, method, impute); + code.append("_fa3.fit(vp_df)"); + + // Display option : Correlation Matrix + if (corrMatrix === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Correlation matrix"); + code.appendLine("display(Markdown('### Correlation matrix'))"); + code.append("display(pd.DataFrame(data= _fa1.corr_ , index=vp_df.columns, columns=vp_df.columns).round(2))"); + } + + // Display option : Scree plot + if (screePlot === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Scree plot"); + code.appendLine("import warnings"); + code.appendLine("with warnings.catch_warnings():"); + code.appendLine(" warnings.simplefilter(action='ignore', category=Warning)"); + code.appendLine(" plt.plot(_fa1.get_factor_variance()[1], 'o-')"); + code.appendLine(" plt.title('Scree Plot')"); + code.appendLine(" plt.xlabel('Factors')"); + code.appendLine(" plt.ylabel('Eigenvalue')"); + code.append(" plt.show()"); + } + + // Communalities + code.appendLine(); + code.appendLine(); + code.appendLine("# Communalities"); + code.appendLine("display(Markdown('### Communalities'))"); + code.appendLine("display(pd.DataFrame(data={'Initial':_fa1.get_communalities(),'Extraction':_fa2.get_communalities()},index=vp_df.columns).round(3))"); + + // Total variance explained + code.appendLine(); + code.appendLine("# Total variance explained"); + code.appendLine("# Initial Eigenvalues"); + code.appendLine("_ss1 = pd.DataFrame(data=_fa1.get_factor_variance(),"); + code.appendLine(" index=[['Initial Eigenvalues' for i in range(3)],['Total','% of variance','Cumulative %']]).T"); + code.appendLine("# Extraction sums of squared loadings"); + code.appendLine("_ss2 = pd.DataFrame(data=_fa1.get_factor_variance(),"); + code.appendLine(" index=[['Extraction sums of squared loadings' for i in range(3)],['Total','% of variance','Cumulative %']]).T[:3]"); + code.appendLine("# Rotation sums of squared loadings"); + code.appendLine("_ss3 = pd.DataFrame(data=_fa3.get_factor_variance(),"); + code.appendLine(" index=[['Rotation sums of squared loadings' for i in range(3)],['Total','% of variance','Cumulative %']]).T"); + code.appendLine(" "); + code.appendLine("display(Markdown('### Total variance explained'))"); + code.appendLine("display(pd.concat([_ss1,_ss2,_ss3], axis=1).round(3))"); + + // Factor Matrix + code.appendLine(); + code.appendLine("# Factor matrix"); + code.appendLine("display(Markdown('### Factor matrix'))"); + code.appendLine("display(pd.DataFrame(data=_fa2.loadings_,index=vp_df.columns,"); + code.appendLine(" columns=list(range(_nof))).round(3))"); + + // Rotated Factor Matrix + code.appendLine(); + code.appendLine("# Rotated factor matrix"); + code.appendLine("display(Markdown('### Rotated factor matrix'))"); + code.appendLine("display(pd.DataFrame(data=_fa3.loadings_,index=vp_df.columns,"); + code.append(" columns=list(range(_nof))).round(3))"); + + codeList.push(code.toString()); + + return codeList; + } + + } + + return FactorAnalysis; +}); \ No newline at end of file diff --git a/visualpython/js/m_stats/LogisticRegression.js b/visualpython/js/m_stats/LogisticRegression.js new file mode 100644 index 00000000..318f63f6 --- /dev/null +++ b/visualpython/js/m_stats/LogisticRegression.js @@ -0,0 +1,193 @@ +/* + * Project Name : Visual Python + * Description : GUI-based Python code generator + * File Name : LogisticRegression.js + * Author : Black Logic + * Note : Correlation Analysis + * License : GNU GPLv3 with Visual Python special exception + * Date : 2023. 06. 02 + * Change Date : + */ + +//============================================================================ +// [CLASS] LogisticRegression +//============================================================================ +define([ + __VP_TEXT_LOADER__('vp_base/html/m_stats/logisticRegression.html'), + 'vp_base/js/com/com_util', + 'vp_base/js/com/com_Const', + 'vp_base/js/com/com_String', + 'vp_base/js/com/com_generatorV2', + 'vp_base/js/com/component/PopupComponent', + 'vp_base/js/com/component/DataSelector', + 'vp_base/js/com/component/MultiSelector', + 'vp_base/js/m_apps/Subset' +], function(eqHTML, com_util, com_Const, com_String, com_generator, PopupComponent, DataSelector, MultiSelector, Subset) { + + /** + * LogisticRegression + */ + class LogisticRegression extends PopupComponent { + _init() { + super._init(); + /** Write codes executed before rendering */ + this.config.sizeLevel = 2; + this.config.checkModules = ['pd']; + this.config.docs = 'https://www.statsmodels.org/stable/api.html'; + + this.state = { + data: '', + dependent: '', + encoding: true, + independent: [], + showOdds: true, + multiCollinearity: true, + ...this.state + }; + + this.subsetEditor = null; + this.columnSelector = null; + } + + _unbindEvent() { + super._unbindEvent(); + $(document).off('change', this.wrapSelector('#dependent')); + } + + _bindEvent() { + super._bindEvent(); + /** Implement binding events */ + var that = this; + + // data change + $(this.wrapSelector('#data')).on('change', function() { + let data = $(this).val(); + that.handleVariableChange(data); + }); + + // dependent change + $(document).on('change', this.wrapSelector('#dependent'), function() { + let depVal = $(this).val(); + that.columnSelector = new MultiSelector(that.wrapSelector('#independent'), + { + mode: 'columns', parent: that.state.data, showDescription: false, + excludeList: [ depVal ] + } + ); + }); + } + + handleVariableChange(data) { + this.state.data = data; + this.state.dependent = ''; + this.state.independent = []; + // render column + com_generator.vp_bindColumnSource(this, 'data', ['dependent'], 'select', false, false); + // render variable selector + this.columnSelector = new MultiSelector(this.wrapSelector('#independent'), + { mode: 'columns', parent: data, showDescription: false } + ); + } + + templateForBody() { + let page = $(eqHTML); + let that = this; + + // generate dataselector + let dataSelector = new DataSelector({ + pageThis: this, id: 'data', placeholder: 'Select data', required: true, boxClasses: 'vp-flex-gap5', + allowDataType: ['DataFrame'], withPopup: false, + finish: function(data, type) { + that.state.data = data; + $(that.wrapSelector('#data')).trigger('change'); + }, + select: function(data, type) { + that.state.data = data; + $(that.wrapSelector('#data')).trigger('change'); + } + }); + $(page).find('#data').replaceWith(dataSelector.toTagString()); + + return page; + } + + render() { + super.render(); + let that = this; + + // render Subset + this.subsetEditor = new Subset({ + pandasObject: '', + config: { name: 'Subset', category: this.name } }, + { + useAsModule: true, + useInputColumns: true, + targetSelector: this.wrapSelector('#data'), + pageThis: this, + finish: function(code) { + $(that.wrapSelector('#data')).val(code); + that.handleVariableChange(code); + } + }); + + // bind column if data exist + if (this.state.data !== '') { + com_generator.vp_bindColumnSource(this, 'data', ['dependent'], 'select', false, false); + } + + let excludeList = []; + if (this.state.dependent !== '') { + excludeList = [ this.state.dependent ]; + } + + // render variable selector + this.columnSelector = new MultiSelector(this.wrapSelector('#independent'), + { mode: 'columns', parent: this.state.data, selectedList: this.state.independent, excludeList: excludeList, showDescription: false }); + } + + generateCode() { + let { data, dependent, encoding, independent, showOdds, multiCollinearity } = this.state; + let codeList = []; + let code = new com_String(); + + let dependentValue = $(this.wrapSelector('#dependent option:selected')).text(); + let independentMulti = this.columnSelector.getDataList(); + this.state.independent = independentMulti; + + // data declaration + code.appendFormatLine("vp_df = {0}.dropna().copy()", data); + if (encoding === true) { + code.appendFormatLine("vp_df['{0}'+'_EL'] = pd.Categorical(vp_df[{1}]).codes", dependentValue, dependent); + dependentValue = dependentValue + '_EL'; + } + code.appendLine(); + code.appendLine("# Logistic regression"); + code.appendLine("from IPython.display import display"); + code.appendLine("import statsmodels.formula.api as smf"); + code.appendFormatLine("_model = smf.logit('{0} ~ {1}', vp_df)", dependentValue, independentMulti.map(x=>x.name).join(' + ')); + code.appendLine("_result = _model.fit()"); + code.appendLine("print(_result.summary())"); + code.appendLine(""); + code.appendLine("# Multi-collinearity statistics"); + code.appendLine("from statsmodels.stats.outliers_influence import variance_inflation_factor"); + code.appendLine("_dfr = pd.DataFrame(_result.summary().tables[1].data[1:],columns=_result.summary().tables[1].data[0]).set_index('')"); + if (showOdds === true) { + code.appendLine("_dfr['Odds'] = np.exp(_result.params)"); + code.appendLine("_dfr['Lower(Odds)'] = np.exp(_result.conf_int()[0])"); + code.appendLine("_dfr['Upper(Odds)'] = np.exp(_result.conf_int()[1])"); + } + if (multiCollinearity === true) { + code.appendLine("for i, col in enumerate(_model.exog_names[1:]):"); + code.appendLine(" _vif = variance_inflation_factor(_model.exog, i+1)"); + code.appendLine(" _dfr.loc[col,'Tolerance'] = 1/_vif"); + code.appendLine(" _dfr.loc[col,'VIF'] = _vif"); + } + code.append("display(_dfr)"); + + return code.toString(); + } + + } + + return LogisticRegression; +}); \ No newline at end of file diff --git a/visualpython/js/m_stats/NormTest.js b/visualpython/js/m_stats/NormTest.js new file mode 100644 index 00000000..0669f6cf --- /dev/null +++ b/visualpython/js/m_stats/NormTest.js @@ -0,0 +1,248 @@ +/* + * Project Name : Visual Python + * Description : GUI-based Python code generator + * File Name : NormTest.js + * Author : Black Logic + * Note : Normality test + * License : GNU GPLv3 with Visual Python special exception + * Date : 2023. 05. 09 + * Change Date : + */ + +//============================================================================ +// [CLASS] NormTest +//============================================================================ +define([ + __VP_TEXT_LOADER__('vp_base/html/m_stats/normTest.html'), + 'vp_base/js/com/com_util', + 'vp_base/js/com/com_Const', + 'vp_base/js/com/com_String', + 'vp_base/js/com/com_generatorV2', + 'vp_base/js/com/component/PopupComponent', + 'vp_base/js/com/component/DataSelector', + 'vp_base/js/m_apps/Subset' +], function(nmHTML, com_util, com_Const, com_String, com_generator, PopupComponent, DataSelector, Subset) { + + /** + * NormTest + */ + class NormTest extends PopupComponent { + _init() { + super._init(); + /** Write codes executed before rendering */ + this.config.sizeLevel = 2; + this.config.checkModules = ['pd']; + this.config.docs = 'https://docs.scipy.org/doc/scipy/reference/'; + + this.state = { + testType: 'shapiro-wilk', + data0: '', + alterHypo: 'two-sided', + histogram: false, + boxplot: false, + qqplot: true, + ...this.state + }; + + this.subsetEditor = {}; + } + + _bindEvent() { + super._bindEvent(); + /** Implement binding events */ + var that = this; + + $(this.wrapSelector('#testType')).on('change', function() { + let testType = $(this).val(); + that.state.testType = testType; + + $(that.wrapSelector('.vp-st-option')).hide(); + $(that.wrapSelector('.vp-st-option.' + testType)).show(); + }); + + $(this.wrapSelector('#data0')).on('change', function() { + if (that.state.data0type === 'DataFrame') { + // DataFrame + that.state.var0 = ''; + $(that.wrapSelector('#var0')).prop('disabled', false); + com_generator.vp_bindColumnSource(that, 'data0', ['var0'], 'select', false, false); + } else { + // Series + that.state.var0 = ''; + $(that.wrapSelector('#var0')).html(''); + $(that.wrapSelector('#var0')).prop('disabled', true); + } + }); + } + + templateForBody() { + let page = $(nmHTML); + let that = this; + + let dataSelector = new DataSelector({ + pageThis: this, id: 'data0', placeholder: 'Select data', required: true, boxClasses: 'vp-flex-gap5', + allowDataType: ['DataFrame', 'Series'], withPopup: false, + finish: function(data, type) { + that.state.data0 = data; + that.state.data0type = type; + that.state.var0 = ''; + $(that.wrapSelector('#data0')).trigger('change'); + }, + select: function(data, type) { + that.state.data0 = data; + that.state.data0type = type; + that.state.var0 = ''; + $(that.wrapSelector('#data0')).trigger('change'); + } + }); + $(page).find('#data0').replaceWith(dataSelector.toTagString()); + + return page; + } + + render() { + super.render(); + let that = this; + + // render Subset + this.subsetEditor['data0'] = new Subset({ + pandasObject: '', + config: { name: 'Subset', category: this.name } }, + { + useAsModule: true, + useInputColumns: true, + targetSelector: this.wrapSelector('#data0'), + pageThis: this, + finish: function(code, state) { + that.state.data0 = code; + $(that.wrapSelector('#data0')).val(code); + that.state.data0type = state.returnType; + $(that.wrapSelector('#data0')).trigger('change'); + } + }); + + // control display option + $(this.wrapSelector('.vp-st-option')).hide(); + $(this.wrapSelector('.vp-st-option.' + this.state.testType)).show(); + } + + generateCode() { + let { testType, data0, data0type, var0, alterHypo, histogram, boxplot, qqplot } = this.state; + let codeList = []; + let code = new com_String(); + + // test type label + let testTypeLabel = $(this.wrapSelector('#testType option:selected')).text(); + code.appendFormatLine('# {0}', testTypeLabel); + + // variable declaration + code.appendFormatLine("vp_df = {0}.dropna().copy()", data0); + + let dataVar = 'vp_df'; + if (var0 !== '') { + dataVar += com_util.formatString("[{0}]", var0); + } + + switch (testType) { + case 'shapiro-wilk': + // 1. Shapiro-wilk test + code.appendLine(); + code.appendLine("# Normality test (Shapiro-Wilk)"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("from scipy import stats"); + code.appendFormatLine("_res = stats.shapiro({0})", dataVar); + code.appendLine("display(Markdown('### Normality test (Shapiro-Wilk)'))"); + code.append("display(pd.DataFrame(data={'Statistic':_res.statistic,'p-value':_res.pvalue},index=['Normality test (Shapiro-Wilk)']))"); + break; + case 'anderson-darling': + // 1. Anderson-Darling test + code.appendLine(); + code.appendLine("# Normality test (Anderson-Darling)"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("from scipy import stats"); + code.appendFormatLine("_res = stats.anderson({0})", dataVar); + code.appendLine("display(Markdown('### Normality test (Anderson-Darling)'))"); + code.appendLine("display(pd.DataFrame(data={'Statistic':[_res.statistic],'Critical values':[_res.critical_values],"); + code.appendLine(" 'Significance level(%)':[_res.significance_level]},"); + code.append(" index=['Normality test (Anderson-Darling)']))"); + break; + case 'kolmogorov-smirnov': + // 1. Kolmogorov-Smirnov test + code.appendLine(); + code.appendLine("# Normality test (Kolmogorov-Smirnov)"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("from scipy import stats"); + code.appendFormatLine("_res = stats.kstest({0}, 'norm', alternative='{1}')", dataVar, alterHypo); + code.appendLine("display(Markdown('### Normality test (Kolmogorov-Smirnov)'))"); + code.appendLine("display(pd.DataFrame(data={'Statistic':_res.statistic,'p-value':_res.pvalue},"); + code.append(" index=['Normality test (Kolmogorov-Smirnov)']))"); + break; + case 'dagostino-pearson': + // 1. D Agostino and Pearson test + code.appendLine(); + code.appendLine("# Normality test (D Agostino and Pearson)"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("from scipy import stats"); + code.appendFormatLine("_res = stats.normaltest({0})", dataVar); + code.appendLine("display(Markdown('### Normality test (D Agostino and Pearson)'))"); + code.appendLine("display(pd.DataFrame(data={'Statistic':_res.statistic,'p-value':_res.pvalue},"); + code.append(" index=['Normality test (D Agostino and Pearson)']))"); + break; + case 'jarque-bera': + // 1. Jarque-Bera test + code.appendLine(); + code.appendLine("# Normality test (Jarque-Bera)"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("from scipy import stats"); + code.appendFormatLine("_res = stats.jarque_bera({0})", dataVar); + code.appendLine("display(Markdown('### Normality test (Jarque-Bera)'))"); + code.appendLine("display(pd.DataFrame(data={'Statistic':_res.statistic,'p-value':_res.pvalue},"); + code.append(" index=['Normality test (Jarque-Bera)']))"); + break; + } + + // Display option + if (histogram === true || boxplot === true || qqplot === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Charts"); + code.appendLine("import seaborn as sns"); + code.appendLine("import warnings"); + code.appendLine("with warnings.catch_warnings():"); + code.append(" warnings.simplefilter(action='ignore', category=Warning)"); + let displayNum = 1; + if (histogram === true) { + code.appendLine(); + code.appendLine(); + code.appendFormatLine(" plt.subplot(2,2,{0})", displayNum++); + code.appendFormatLine(" sns.histplot({0}, stat='density', kde=True)", dataVar); + code.append(" plt.title('Histogram')"); + } + if (boxplot === true) { + code.appendLine(); + code.appendLine(); + code.appendFormatLine(" plt.subplot(2,2,{0})", displayNum++); + code.appendFormatLine(" sns.boxplot(y={0})", dataVar); + code.append(" plt.title('Boxplot')"); + } + + if (qqplot === true) { + code.appendLine(); + code.appendLine(); + code.appendFormatLine(" plt.subplot(2,2,{0})", displayNum); + code.appendFormatLine(" stats.probplot({0}, plot=plt)", dataVar); + code.append(" plt.title('Q-Q Plot')"); + } + code.appendLine(); + code.appendLine(); + code.appendLine(" plt.tight_layout()"); + code.append(" plt.show()"); + } + + return code.toString(); + } + + } + + return NormTest; +}); \ No newline at end of file diff --git a/visualpython/js/m_stats/ProbDist.js b/visualpython/js/m_stats/ProbDist.js new file mode 100644 index 00000000..245cd9f6 --- /dev/null +++ b/visualpython/js/m_stats/ProbDist.js @@ -0,0 +1,440 @@ +/* + * Project Name : Visual Python + * Description : GUI-based Python code generator + * File Name : ProbDist.js + * Author : Black Logic + * Note : Probability Distribution + * License : GNU GPLv3 with Visual Python special exception + * Date : 2023. 05. 09 + * Change Date : + */ + +//============================================================================ +// [CLASS] ProbDist +//============================================================================ +define([ + __VP_TEXT_LOADER__('vp_base/html/m_stats/probDist.html'), + __VP_CSS_LOADER__('vp_base/css/m_stats/probDist'), + 'vp_base/js/com/com_util', + 'vp_base/js/com/com_Const', + 'vp_base/js/com/com_String', + 'vp_base/js/com/com_generatorV2', + 'vp_base/data/m_stats/statsLibrary', + 'vp_base/js/com/component/PopupComponent', + 'vp_base/js/com/component/DataSelector' +], function(pdHTML, pdCss, com_util, com_Const, com_String, com_generator, STATS_LIBRARIES, PopupComponent, DataSelector) { + + /** + * ProbDist + */ + class ProbDist extends PopupComponent { + _init() { + super._init(); + /** Write codes executed before rendering */ + this.config.sizeLevel = 2; + this.config.checkModules = ['pd', 'plt']; + this.config.docs = 'https://docs.scipy.org/doc/scipy/reference/'; + this.config.helpview = true; + + this.state = { + distType: 'normal', + userOption: '', + action: 'random-number', + // random-number + size: 10000, + randomState: '', + allocateTo: 'samples', + sampledDist: true, + // distribution-plot + probDensityFunc: true, + probMassFunc: true, + cumDistFunc: true, + // stats-to-pvalue + stats: '', + pAlter: 'two-sided', + // pvalue-to-stats + pvalue: '', + statsAlter: 'two-sided', + ...this.state + }; + + this.distList = [ + { + label: 'Discrete probability distribution', + child: ['bernoulli', 'binomial', 'multinomial'] + }, + { + label: 'Continuous probability distribution', + child: ['uniform','normal','beta','gamma','studentst','chi2','f','dirichlet','multivariate_normal'] + } + ]; + } + + _bindEvent() { + super._bindEvent(); + /** Implement binding events */ + var that = this; + + $(this.wrapSelector('#distType')).on('change', function() { + let distType = $(this).val(); + that.state.distType = distType; + $(that.wrapSelector('.vp-pd-dist-option-box')).html(that.templateForOption(distType)); + + $(that.wrapSelector('.vp-pd-display-option')).hide(); + // show/hide display option + if (that.distList[0].child.includes(distType)) { + // discrete option + $(that.wrapSelector('.vp-pd-display-option.dist')).show(); + + // set size to 1000 + $(that.wrapSelector('#size')).val(1000); + that.state.size = 1000; + + // hide distribution plot for multinomial + if (distType === 'multinomial') { + $(that.wrapSelector('.vp-pd-display-option.dist-plot')).hide(); + // hide other actions + if (that.state.action !== 'random-number') { + $(that.wrapSelector('#action')).val('random-number'); + $(that.wrapSelector('#action')).trigger('change'); + } + } else { + // hide continuous action + if (that.state.action === 'stats-to-pvalue' || that.state.action === 'pvalue-to-stats') { + $(that.wrapSelector('#action')).val('random-number'); + $(that.wrapSelector('#action')).trigger('change'); + } + } + } else { + // continuous option + $(that.wrapSelector('.vp-pd-display-option.cont')).show(); + + // set size to 10000 + $(that.wrapSelector('#size')).val(10000); + that.state.size = 10000; + } + + // show install button + let thisDistObj = STATS_LIBRARIES[distType]; + if (thisDistObj.install != undefined) { + $(that.wrapSelector('#vp_installLibrary')).show(); + } else { + $(that.wrapSelector('#vp_installLibrary')).hide(); + } + + // set help content + that.setHelpContent(thisDistObj.help); + }); + + $(this.wrapSelector('#action')).on('change', function() { + let action = $(this).val(); + that.state.action = action; + + $(that.wrapSelector('.vp-pd-action-box')).hide(); + $(that.wrapSelector('.vp-pd-action-box.' + action)).show(); + + $(that.wrapSelector('.vp-pd-display-option')).hide(); + // show/hide display option + if (that.distList[0].child.includes(that.state.distType)) { + // discrete option + $(that.wrapSelector('.vp-pd-display-option.dist')).show(); + } else { + // continuous option + $(that.wrapSelector('.vp-pd-display-option.cont')).show(); + } + }); + } + + templateForBody() { + let page = $(pdHTML); + let that = this; + + //================================================================ + // Distribution type creation + //================================================================ + // dist types + let distTypeTag = new com_String(); + this.distList.forEach(distObj => { + let { label, child } = distObj; + let distOptionTag = new com_String(); + child && child.forEach(opt => { + let optConfig = STATS_LIBRARIES[opt]; + let selectedFlag = ''; + if (opt == that.state.distType) { + selectedFlag = 'selected'; + } + distOptionTag.appendFormatLine('', + opt, selectedFlag, optConfig.name); + }) + distTypeTag.appendFormatLine('{1}', + label, distOptionTag.toString()); + }); + $(page).find('#distType').html(distTypeTag.toString()); + + // render option page + $(page).find('.vp-pd-dist-option-box').html(this.templateForOption(this.state.distType)); + + // control display option + $(this.wrapSelector('.vp-pd-display-option')).hide(); + // show/hide display option + if (this.distList[0].child.includes(this.state.distType)) { + // discrete option + $(this.wrapSelector('.vp-pd-display-option.dist')).show(); + } else { + // continuous option + $(this.wrapSelector('.vp-pd-display-option.cont')).show(); + } + + return page; + } + + templateForOption(distType) { + let config = STATS_LIBRARIES[distType]; + let state = this.state; + + let optBox = new com_String(); + // render tag + config.options.forEach(opt => { + optBox.appendFormatLine('{3}' + , opt.name, (opt.required===true?'vp-orange-text':''), opt.name, com_util.optionToLabel(opt.name)); + let content = com_generator.renderContent(this, opt.component[0], opt, state); + optBox.appendLine(content[0].outerHTML); + }); + // render user option + optBox.appendFormatLine('{1}', 'userOption', 'User option'); + optBox.appendFormatLine('', + 'userOption', 'key=value, ...', this.state.userOption); + return optBox.toString(); + } + + render() { + super.render(); + + let allocateSelector = new DataSelector({ + type: 'data', + pageThis: this, + id: 'allocatedTo', + classes: 'vp-input vp-state', + placeholder: '_res', + finish: function() { + ; + } + }); + $(this.wrapSelector('#allocatedTo')).replaceWith(allocateSelector.toTagString()); + + this.setHelpContent(STATS_LIBRARIES[this.state.distType].help, true, 'import scipy.stats as _vp_stats'); + } + + generateCode() { + this.config.checkModules = ['pd']; + let { + distType, userOption, action, + size, randomState, allocateTo, sampledDist, + probDensityFunc, probMassFunc, cumDistFunc, + stats, pAlter, + pvalue, statsAlter + } = this.state; + + let codeList = []; + let code = new com_String(); + /** + * Model Creation + */ + let config = STATS_LIBRARIES[distType]; + let label = config.name; + code.appendLine(config.import); + code.appendLine(); + + // model code + let modelCode = config.code; + modelCode = com_generator.vp_codeGenerator(this, config, this.state, (userOption != ''? ', ' + userOption : '')); + code.append(modelCode); + + switch (action) { + case 'random-number': + code.appendLine(); + code.appendLine(); + code.appendFormatLine("# Generate random numbers ({0})", label); + code.appendLine("from IPython.display import display"); + code.appendFormat('{0} = _rv.rvs(size={1}', allocateTo, size); + if (randomState !== '') { + code.appendFormat(", random_state={0}", randomState); + } + code.appendLine(')'); + code.appendFormat("display({0})", allocateTo); + + if (sampledDist === true) { + this.addCheckModules('plt'); + this.addCheckModules('sns'); + + code.appendLine(); + code.appendLine(); + code.appendFormatLine("# Sample distribution ({0})", label); + code.appendLine("import warnings"); + code.appendLine("with warnings.catch_warnings():"); + code.appendLine(" warnings.simplefilter(action='ignore', category=Warning)"); + if (distType === 'multinomial') { + // code.appendFormatLine(" plt.boxplot(x={0})", allocateTo); + code.appendFormatLine(" for i in range(0, {0}.shape[1]):", allocateTo); + code.appendLine(" plt.subplot(2, 2, i+1)"); + code.appendLine(" plt.title('$x$=' + str(i))"); + code.appendFormatLine(" sns.countplot(x=[ x[i] for x in {0} ])", allocateTo); + code.appendLine(" plt.suptitle('Generate random numbers: Multinomial')"); + code.appendLine(" plt.tight_layout()"); + code.appendLine(" plt.show()"); + } else { + if (this.distList[0].child.includes(distType)) { + code.appendFormatLine(" sns.countplot(x={0})", allocateTo); + } else { + code.appendFormatLine(" sns.histplot({0}, stat='density', kde=True)", allocateTo); + } + code.appendFormatLine(" plt.title('Generate random numbers: {0}')", label.replace("'", "\\'")); + code.appendLine(" plt.xlabel('$x$')"); + code.append(" plt.show()"); + } + } + break; + case 'distribution-plot': + if (this.distList[0].child.includes(distType)) { + if (probMassFunc === true) { + this.addCheckModules('np'); + this.addCheckModules('plt'); + + code.appendLine(); + code.appendLine(); + code.appendFormatLine("# Probability mass function ({0})", label); + if (distType === 'bernoulli') { + code.appendLine("plt.bar([0,1], _rv.pmf([0,1]))"); + code.appendFormatLine("plt.title('Probability mass function: {0}')", label.replace("'", "\\'")); + code.appendLine("plt.xlim(-1, 2)"); + code.appendLine("plt.ylim(0, 1)"); + code.appendLine("plt.xticks([0, 1], ['x=0', 'x=1'])"); + code.appendLine("plt.xlabel('$x$')"); + code.appendLine("plt.ylabel('$p(x)$')"); + code.append("plt.show()"); + } else if (distType === 'binomial') { + var { n=10 } = this.state; + code.appendFormatLine("plt.bar(range(0,{0}), _rv.pmf(range(0,{1})))", n, n); + code.appendFormatLine("plt.title('Probability mass function: {0}')", label.replace("'", "\\'")); + code.appendFormatLine("plt.xlim(-1, {0})", n); + code.appendFormatLine("plt.xticks(range(0, {0}), ['x='+str(i) for i in range(0, {1})])", n, n); + code.appendLine("plt.xlabel('$x$')"); + code.appendLine("plt.ylabel('$p(x)$')"); + code.append("plt.show()"); + } else if (distType === 'multinomial') { + code.appendFormatLine("for i in range(0, {0}.shape[1]):", allocateTo); + code.appendLine(" plt.subplot(2, 2, i+1)"); + code.appendLine(" plt.title('$x$=' + str(i))"); + code.appendFormatLine(" sns.countplot(x=[ x[i] for x in {0} ])", allocateTo); + code.appendLine("plt.suptitle('Probability mass function: Multinomial')"); + code.appendLine("plt.tight_layout()"); + code.append("plt.show()"); + } + } + } else { + let start = -5; + let end = 5; + switch (distType) { + case 'normal': + case 'studentst': + case 'multivariate_normal': + start = -5; + end = 5; + break; + case 'uniform': + case 'beta': + case 'dirichlet': + start = 0; + end = 1; + break; + case 'gamma': + case 'chi2': + start = 0; + end = 30; + break; + case 'f': + start = 0; + end = 10; + break; + } + + if (probDensityFunc === true || cumDistFunc === true) { + code.appendLine(); + code.appendFormat("x = np.linspace({0}, {1}, 100)", start, end); + if (probDensityFunc === true) { + this.addCheckModules('np'); + this.addCheckModules('plt'); + + code.appendLine(); + code.appendLine(); + code.appendFormatLine("# Probability density function ({0})", label); + code.appendLine("plt.plot(x, _rv.pdf(x))"); + code.appendFormatLine("plt.title('Probability density function: {0}')", label.replace("'", "\\'")); + code.appendLine("plt.xlabel('$x$')"); + code.appendLine("plt.ylabel('$p(x)$')"); + code.append("plt.show()"); + } + if (cumDistFunc === true) { + this.addCheckModules('np'); + this.addCheckModules('plt'); + + code.appendLine(); + code.appendLine(); + code.appendFormatLine("# Cumulative distribution function ({0})", label); + code.appendLine("import warnings"); + code.appendLine("with warnings.catch_warnings():"); + code.appendFormatLine(" _x = np.linspace({0}, {1}, 100)", start, end); + code.appendLine(" plt.plot(_x, _rv.cdf(_x))"); + code.appendLine(); + code.appendFormatLine(" plt.title('Cumulative distribution function: {0}')", label.replace("'", "\\'")); + code.appendLine(" plt.xlabel('$x$')"); + code.appendLine(" plt.ylabel('$F(x)$')"); + code.append(" plt.show()"); + } + } + } + break; + case 'stats-to-pvalue': + if (pAlter === 'one-sided') { + // one-sided + code.appendLine(); + code.appendLine(); + code.appendLine("# Proportional values"); + code.appendFormatLine("p_value = _rv.sf(abs({0}))", stats); + code.append("p_value"); + } else { + // two-sided + code.appendLine(); + code.appendLine(); + code.appendLine("# Proportional values"); + code.appendFormatLine("p_value = _rv.sf(abs({0}))*2", stats); + code.append("p_value"); + } + break; + case 'pvalue-to-stats': + if (statsAlter === 'one-sided') { + // one-sided + code.appendLine(); + code.appendLine(); + code.appendLine("# Statistic"); + code.appendFormatLine("statistic = _rv.isf({0})", pvalue); + code.append("statistic"); + } else { + // two-sided + code.appendLine(); + code.appendLine(); + code.appendLine("# Statistic"); + code.appendFormatLine("statistic = _rv.isf({0}/2)", pvalue); + code.append("statistic"); + } + break; + } + codeList.push(code.toString()); + + return codeList; + } + + } + + return ProbDist; +}); \ No newline at end of file diff --git a/visualpython/js/m_stats/Regression.js b/visualpython/js/m_stats/Regression.js new file mode 100644 index 00000000..a38c811b --- /dev/null +++ b/visualpython/js/m_stats/Regression.js @@ -0,0 +1,789 @@ +/* + * Project Name : Visual Python + * Description : GUI-based Python code generator + * File Name : Regression.js + * Author : Black Logic + * Note : Equal Variance test + * License : GNU GPLv3 with Visual Python special exception + * Date : 2023. 05. 09 + * Change Date : + */ + +//============================================================================ +// [CLASS] EqualVarTest +//============================================================================ +define([ + __VP_TEXT_LOADER__('vp_base/html/m_stats/regression.html'), + __VP_CSS_LOADER__('vp_base/css/m_stats/regression'), + 'vp_base/js/com/com_util', + 'vp_base/js/com/com_Const', + 'vp_base/js/com/com_String', + 'vp_base/js/com/com_generatorV2', + 'vp_base/js/com/component/PopupComponent', + 'vp_base/js/com/component/DataSelector', + 'vp_base/js/com/component/MultiSelector', + 'vp_base/js/m_apps/Subset' +], function(eqHTML, rgCss, com_util, com_Const, com_String, com_generator, PopupComponent, DataSelector, MultiSelector, Subset) { + + /** + * Regression + */ + class Regression extends PopupComponent { + _init() { + super._init(); + /** Write codes executed before rendering */ + this.config.sizeLevel = 2; + this.config.checkModules = ['pd']; + this.config.docs = 'https://www.statsmodels.org/stable/api.html'; + + this.state = { + testType: 'simple', + // Data selection + data: '', + dataType: '', + dependent: '', + independent: '', + independentMulti: [], + moderated: '', + mediated: '', + // options + categorical: [], + method: 'enter', + meanCentering: true, + sobelTest: true, + // Multi-collinearity + multiCollinearity: true, + // Residual option + statistics: false, + normTest: true, + histogram: true, + scatterplot: true, + rmse: false, + ...this.state + }; + + this.colBindList = ['dependent', 'independent', 'moderated', 'mediated']; + + this.subsetEditor = null; + this.columnSelector = null; + } + + _unbindEvent() { + super._unbindEvent(); + $(document).off('change', this.wrapSelector('#dependent')); + } + + _bindEvent() { + super._bindEvent(); + /** Implement binding events */ + var that = this; + + // change test type + $(this.wrapSelector('#testType')).on('change', function() { + let testType = $(this).val(); + that.state.testType = testType; + + $(that.wrapSelector('.vp-st-option')).hide(); + $(that.wrapSelector('.vp-st-option.' + testType)).show(); + + let excludeList = []; + if (that.state.testType === 'multiple' + || that.state.testType === 'hierarchical' + || that.state.testType === 'dummy') { + let depVal = $(that.wrapSelector('#dependent')).val(); + excludeList = [ depVal ]; + } + + // render variable selector + that.columnSelector = new MultiSelector(that.wrapSelector('#independentBox'), + { + mode: 'columns', parent: that.state.data, showDescription: false, + excludeList: excludeList, + change: function(type, list) { + that._handleMultiColumnChange(type, list); + } + }); + }); + + // data change + $(this.wrapSelector('#data')).on('change', function() { + let data = $(this).val(); + that.handleVariableChange(data); + }); + + // dependent change + $(document).on('change', this.wrapSelector('#dependent'), function() { + let depVal = $(this).val(); + if (that.state.testType === 'multiple' + || that.state.testType === 'hierarchical' + || that.state.testType === 'dummy') { + that.columnSelector = new MultiSelector(that.wrapSelector('#independentBox'), + { + mode: 'columns', parent: that.state.data, showDescription: false, + excludeList: [ depVal ], + change: function(type, list) { + that._handleMultiColumnChange(type, list); + } + } + ); + } + }); + } + + handleVariableChange(data) { + this.state.data = data; + this.state.independentMulti = []; + let that = this; + // bind column sources + if (this.state.dataType === 'DataFrame') { + // DataFrame + this.colBindList && this.colBindList.forEach(id => { + that.state[id] = ''; + $(that.wrapSelector('#' + id)).prop('disabled', false); + }); + com_generator.vp_bindColumnSource(this, 'data', this.colBindList, 'select', false, false); + } else { + // Others + this.colBindList && this.colBindList.forEach(id => { + that.state[id] = ''; + $(that.wrapSelector('#' + id)).html(''); + $(that.wrapSelector('#' + id)).prop('disabled', true); + }); + } + // render variable selector + this.columnSelector = new MultiSelector(this.wrapSelector('#independentBox'), + { + mode: 'columns', parent: data, showDescription: false, + change: function(type, list) { + that._handleMultiColumnChange(type, list); + } + } + ); + } + + _handleMultiColumnChange(type, list) { + let $newCateBox = $('

'); + let that = this; + list && list.forEach(item => { + let checkedStr = 'checked'; + if ($(that.wrapSelector('.vp-categorical-box input[data-name="' + item.name + '"]')).length > 0) { + $(that.wrapSelector('.vp-categorical-box input[data-name="' + item.name + '"]')).prop('checked')?'checked':''; + } + $newCateBox.append(` + ${item.name} + `); + }); + $(this.wrapSelector('.vp-categorical-box')).replaceWith($newCateBox); + } + + templateForBody() { + let page = $(eqHTML); + let that = this; + + let dataSelector = new DataSelector({ + pageThis: this, id: 'data', placeholder: 'Select data', required: true, boxClasses: 'vp-flex-gap5', + allowDataType: ['DataFrame'], withPopup: false, + finish: function(data, type) { + that.state.data = data; + that.state.dataType = type; + $(that.wrapSelector('#data')).trigger('change'); + }, + select: function(data, type) { + that.state.data = data; + that.state.dataType = type; + $(that.wrapSelector('#data')).trigger('change'); + } + }); + $(page).find('#data').replaceWith(dataSelector.toTagString()); + + // depend on test type + $(page).find('.vp-st-option').hide(); + $(page).find('.vp-st-option.' + this.state.testType).show(); + + return page; + } + + render() { + super.render(); + let that = this; + + // render Subset + this.subsetEditor = new Subset({ + pandasObject: this.state.data, + config: { name: 'Subset', category: this.name } }, + { + useAsModule: true, + useInputColumns: true, + targetSelector: this.wrapSelector('#data'), + pageThis: this, + finish: function(code, state) { + that.state.data = code; + $(that.wrapSelector('#data')).val(code); + that.state.dataType = state.returnType; + $(that.wrapSelector('#data')).trigger('change'); + } + }); + + let excludeList = []; + if (this.state.testType === 'multiple' + || this.state.testType === 'hierarchical' + || this.state.testType === 'dummy') { + if (this.state.dependent !== '') { + excludeList = [ this.state.dependent ]; + } + } + + // render variable selector + this.columnSelector = new MultiSelector(this.wrapSelector('#indenpendentBox'), + { + mode: 'columns', parent: this.state.data, + selectedList: this.state.independentMulti.map(x=>x.code), excludeList: excludeList, showDescription: false + } + ); + + // bind column if data exist + if (this.state.data !== '') { + com_generator.vp_bindColumnSource(this, 'data', this.colBindList, 'select', false, false); + } + + // control display option + $(this.wrapSelector('.vp-st-option')).hide(); + $(this.wrapSelector('.vp-st-option.' + this.state.testType)).show(); + } + + generateCode() { + let { testType, + // Data selection + data, dataType, dependent, independent, independentMulti, moderated, mediated, + // options + method, meanCentering, sobelTest, + // Multi-collinearity + multiCollinearity, + // Residual option + statistics, normTest, histogram, scatterplot, rmse, + } = this.state; + let codeList = []; + let code = new com_String(); + let that = this; + let lastModelNum = 0; + + // Commentary + let testTypeLabel = $(this.wrapSelector('#testType option:selected')).text(); + let methodLabel = $(this.wrapSelector('#method option:selected')).text(); + if (testType === 'multiple') { + code.appendFormatLine("# {0} > Method: {1}", testTypeLabel, methodLabel); + } else { + code.appendFormatLine("# {0}", testTypeLabel); + } + + // data declaration + code.appendFormatLine("vp_df = {0}.dropna().copy()", data); + + // data and columns + let dependentValue = $(this.wrapSelector('#dependent option:selected')).text(); + let independentValue = $(this.wrapSelector('#independent option:selected')).text(); + let moderatedValue = $(this.wrapSelector('#moderated option:selected')).text(); + let mediatedValue = $(this.wrapSelector('#mediated option:selected')).text(); + independentMulti = this.columnSelector.getDataList(); + this.state.independentMulti = independentMulti; + + switch (testType) { + case 'simple': + // 1. Simple + code.appendLine(); + code.appendLine("# Simple linear regression"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("import statsmodels.formula.api as smf"); + code.appendLine("# Model - Dependent variable ~ Independent variable"); + code.appendFormatLine("_model = smf.ols('{0} ~ {1}', vp_df)", dependentValue, independentValue); + code.appendLine("_result = _model.fit()"); + code.appendLine("display(Markdown('### Model - Dependent variable ~ Independent variable'))"); + code.append("print(_result.summary())"); + // Multi-collinearity statistics + if (multiCollinearity === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Multi-collinearity statistics"); + code.appendLine("from statsmodels.stats.outliers_influence import variance_inflation_factor"); + code.appendLine("_dfr = pd.DataFrame(_result.summary().tables[1].data[1:],columns=_result.summary().tables[1].data[0]).set_index('')"); + code.appendLine("for i, col in enumerate(_model.exog_names[1:]):"); + code.appendLine(" _vif = variance_inflation_factor(_model.exog, i+1)"); + code.appendLine(" _dfr.loc[col,'Tolerance'] = 1/_vif"); + code.appendLine(" _dfr.loc[col,'VIF'] = _vif"); + code.append("display(_dfr)"); + } + break; + case 'multiple': + // 2. Multiple + code.appendLine(); + if (method === 'enter') { + code.appendLine("# Model - Dependent variable ~ Independent variable"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("import statsmodels.formula.api as smf"); + code.appendFormatLine("_model = smf.ols('{0} ~ {1}', vp_df)", dependentValue, independentMulti.map(x => x.name).join(' + ')); + code.appendLine("_result = _model.fit()"); + code.appendLine("display(Markdown('### Model - Dependent variable ~ Independent variable'))"); + code.append("print(_result.summary())"); + } else if (method === 'stepwise') { + // Inner function : vp_stepwise_select + this.addCheckModules('sm'); + this.addCheckModules('vp_stepwise_select'); + + code.appendFormatLine("_selected_stepwise = vp_stepwise_select(vp_df[[{0}]], vp_df[{1}])", independentMulti.map(x => x.code).join(','), dependent); + code.appendLine(""); + code.appendLine("# Model 1 - Dependent variable ~ Independent variable"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("import statsmodels.api as sm"); + code.appendFormatLine("_model = sm.OLS(vp_df[{0}], sm.add_constant(vp_df[_selected_stepwise[0]]))", dependent); + code.appendLine("_result = _model.fit()"); + code.appendLine("display(Markdown('### Model 1 - Dependent variable ~ Independent variable'))"); + code.append("print(_result.summary())"); + if (multiCollinearity === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 1 - Multi-collinearity statistics"); + code.appendLine("from statsmodels.stats.outliers_influence import variance_inflation_factor"); + code.appendLine("_dfr = pd.DataFrame(_result.summary().tables[1].data[1:],columns=_result.summary().tables[1].data[0]).set_index('')"); + code.appendLine("for i, col in enumerate(_model.exog_names[1:]):"); + code.appendLine(" _vif = variance_inflation_factor(_model.exog, i+1)"); + code.appendLine(" _dfr.loc[col,'Tolerance'] = 1/_vif"); + code.appendLine(" _dfr.loc[col,'VIF'] = _vif"); + code.append("display(_dfr)"); + } + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 2 - Dependent variable ~ Stepwised variable"); + code.appendLine("import statsmodels.api as sm"); + code.appendFormatLine("_model = sm.OLS(vp_df[{0}], sm.add_constant(vp_df[_selected_stepwise]))", dependent); + code.appendLine("_result = _model.fit()"); + code.appendLine("display(Markdown('### Model 2 - Dependent variable ~ Stepwised variable'))"); + code.append("print(_result.summary())"); + if (multiCollinearity === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 2 - Multi-collinearity statistics"); + code.appendLine("from statsmodels.stats.outliers_influence import variance_inflation_factor"); + code.appendLine("_dfr = pd.DataFrame(_result.summary().tables[1].data[1:],columns=_result.summary().tables[1].data[0]).set_index('')"); + code.appendLine("for i, col in enumerate(_model.exog_names[1:]):"); + code.appendLine(" _vif = variance_inflation_factor(_model.exog, i+1)"); + code.appendLine(" _dfr.loc[col,'Tolerance'] = 1/_vif"); + code.appendLine(" _dfr.loc[col,'VIF'] = _vif"); + code.append("display(_dfr)"); + } + // set last model number + lastModelNum = 2; + } else if (method === 'backward') { + // Inner function : vp_backward_select + this.addCheckModules('sm'); + this.addCheckModules('vp_backward_select'); + + code.appendFormatLine("_selected_backward = vp_backward_select(vp_df[[{0}]], vp_df[{1}])", independentMulti.map(x => x.code).join(','), dependent); + code.appendLine(); + code.appendLine("# Model 1 - Dependent variable ~ Independent variable"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("import statsmodels.api as sm"); + code.appendFormatLine("_model = sm.OLS(vp_df[{0}], sm.add_constant(vp_df[[{1}]]))", dependent, independentMulti.map(x => x.code).join(',')); + code.appendLine("_result = _model.fit()"); + code.appendLine("display(Markdown('### Model 1 - Dependent variable ~ Independent variable'))"); + code.append("print(_result.summary())"); + if (multiCollinearity === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 1 - Multi-collinearity statistics"); + code.appendLine("from statsmodels.stats.outliers_influence import variance_inflation_factor"); + code.appendLine("_dfr = pd.DataFrame(_result.summary().tables[1].data[1:],columns=_result.summary().tables[1].data[0]).set_index('')"); + code.appendLine("for i, col in enumerate(_model.exog_names[1:]):"); + code.appendLine(" _vif = variance_inflation_factor(_model.exog, i+1)"); + code.appendLine(" _dfr.loc[col,'Tolerance'] = 1/_vif"); + code.appendLine(" _dfr.loc[col,'VIF'] = _vif"); + code.append("display(_dfr)"); + } + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 2 - Dependent variable ~ Backward variable"); + code.appendLine("import statsmodels.api as sm"); + code.appendFormatLine("_model = sm.OLS(vp_df[{0}], sm.add_constant(vp_df[_selected_backward]))", dependent); + code.appendLine("_result = _model.fit()"); + code.appendLine("display(Markdown('### Model 2 - Dependent variable ~ Backward variable'))"); + code.append("print(_result.summary())"); + if (multiCollinearity === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 2 - Multi-collinearity statistics"); + code.appendLine("from statsmodels.stats.outliers_influence import variance_inflation_factor"); + code.appendLine("_dfr = pd.DataFrame(_result.summary().tables[1].data[1:],columns=_result.summary().tables[1].data[0]).set_index('')"); + code.appendLine("for i, col in enumerate(_model.exog_names[1:]):"); + code.appendLine(" _vif = variance_inflation_factor(_model.exog, i+1)"); + code.appendLine(" _dfr.loc[col,'Tolerance'] = 1/_vif"); + code.appendLine(" _dfr.loc[col,'VIF'] = _vif"); + code.append("display(_dfr)"); + } + // set last model number + lastModelNum = 2; + } else if (method === 'forward') { + // Inner function : vp_forward_select + this.addCheckModules('sm'); + this.addCheckModules('vp_forward_select'); + + code.appendFormatLine("_selected_forward = vp_forward_select(vp_df[[{0}]], vp_df[{1}])", independentMulti.map(x => x.code).join(','), dependent); + code.appendLine(); + code.appendLine("# Model 1 - Dependent variable ~ Independent variable"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("import statsmodels.api as sm"); + code.appendFormatLine("_model = sm.OLS(vp_df[{0}], sm.add_constant(vp_df[_selected_forward[0]]))", dependent); + code.appendLine("_result = _model.fit()"); + code.appendLine("display(Markdown('### Model 1 - Dependent variable ~ Independent variable'))"); + code.append("print(_result.summary())"); + if (multiCollinearity === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 1 - Multi-collinearity statistics"); + code.appendLine("from statsmodels.stats.outliers_influence import variance_inflation_factor"); + code.appendLine("_dfr = pd.DataFrame(_result.summary().tables[1].data[1:],columns=_result.summary().tables[1].data[0]).set_index('')"); + code.appendLine("for i, col in enumerate(_model.exog_names[1:]):"); + code.appendLine(" _vif = variance_inflation_factor(_model.exog, i+1)"); + code.appendLine(" _dfr.loc[col,'Tolerance'] = 1/_vif"); + code.appendLine(" _dfr.loc[col,'VIF'] = _vif"); + code.append("display(_dfr)"); + } + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 2 - Dependent variable ~ Forward variable"); + code.appendLine("import statsmodels.api as sm"); + code.appendFormatLine("_model = sm.OLS(vp_df[{0}], sm.add_constant(vp_df[_selected_forward]))", dependent); + code.appendLine("_result = _model.fit()"); + code.appendLine("display(Markdown('### Model 2 - Dependent variable ~ Forward variable'))"); + code.append("print(_result.summary())"); + if (multiCollinearity === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 2 - Multi-collinearity statistics"); + code.appendLine("from statsmodels.stats.outliers_influence import variance_inflation_factor"); + code.appendLine("_dfr = pd.DataFrame(_result.summary().tables[1].data[1:],columns=_result.summary().tables[1].data[0]).set_index('')"); + code.appendLine("for i, col in enumerate(_model.exog_names[1:]):"); + code.appendLine(" _vif = variance_inflation_factor(_model.exog, i+1)"); + code.appendLine(" _dfr.loc[col,'Tolerance'] = 1/_vif"); + code.appendLine(" _dfr.loc[col,'VIF'] = _vif"); + code.append("display(_dfr)"); + } + // set last model number + lastModelNum = 2; + } + break; + case 'hierarchical': + // 3. Hierarchical + for (let i = 0; i < independentMulti.length; i++) { + if (i === 0) { + code.appendLine(); + } else { + code.appendLine(); + code.appendLine(); + } + code.appendFormatLine("# Model {0} - Hierarchical linear regression", (i + 1)); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("import statsmodels.formula.api as smf"); + code.appendFormatLine("_model = smf.ols('{0} ~ {1}', vp_df)", dependentValue, independentMulti.slice(0, i + 1).map(x => x.name).join(' + ')); + code.appendLine("_result = _model.fit()"); + code.appendFormatLine("display(Markdown('### Model {0} - Dependent variable ~ Independent variable'))", (i + 1)); + code.append("print(_result.summary())"); + if (multiCollinearity === true) { + code.appendLine(); + code.appendLine(); + code.appendFormatLine("# Model {0} - Multi-collinearity statistics", (i + 1)); + code.appendLine("from statsmodels.stats.outliers_influence import variance_inflation_factor"); + code.appendLine("_dfr = pd.DataFrame(_result.summary().tables[1].data[1:],columns=_result.summary().tables[1].data[0]).set_index('')"); + code.appendLine("for i, col in enumerate(_model.exog_names[1:]):"); + code.appendLine(" _vif = variance_inflation_factor(_model.exog, i+1)"); + code.appendLine(" _dfr.loc[col,'Tolerance'] = 1/_vif"); + code.appendLine(" _dfr.loc[col,'VIF'] = _vif"); + code.append("display(_dfr)"); + } + } + lastModelNum = independentMulti.length; + break; + case 'moderated': + // 4. Moderated + // Mean centering + if (meanCentering === true) { + code.appendLine(); + code.appendLine("# Mean Centering "); + independentValue = com_util.formatString("{0}_MC", independentValue); + moderatedValue = com_util.formatString("{0}_MC", moderatedValue); + code.appendFormatLine("vp_df['{0}'] = vp_df[{1}] - vp_df[{2}].mean(numeric_only=True)", independentValue, independent, independent); + code.appendFormatLine("vp_df['{0}'] = vp_df[{1}] - vp_df[{2}].mean(numeric_only=True)", moderatedValue, moderated, moderated); + } + // Model 1 to 3 + code.appendLine(); + code.appendLine("# Model 1 - Dependent variable ~ Independent variable"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("import statsmodels.formula.api as smf"); + code.appendFormatLine("_model = smf.ols('{0} ~ {1}', vp_df)", dependentValue, independentValue); + code.appendLine("_result = _model.fit()"); + code.appendLine("display(Markdown('### Model 1 - Dependent variable ~ Independent variable'))"); + code.append("print(_result.summary())"); + if (multiCollinearity === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 1 - Multi-collinearity statistics"); + code.appendLine("from statsmodels.stats.outliers_influence import variance_inflation_factor"); + code.appendLine("_dfr = pd.DataFrame(_result.summary().tables[1].data[1:],columns=_result.summary().tables[1].data[0]).set_index('')"); + code.appendLine("for i, col in enumerate(_model.exog_names[1:]):"); + code.appendLine(" _vif = variance_inflation_factor(_model.exog, i+1)"); + code.appendLine(" _dfr.loc[col,'Tolerance'] = 1/_vif"); + code.appendLine(" _dfr.loc[col,'VIF'] = _vif"); + code.append("display(_dfr)"); + } + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 2 - Dependent variable ~ Independent variable + Moderated variable"); + code.appendLine("import statsmodels.formula.api as smf"); + code.appendFormatLine("_model = smf.ols('{0} ~ {1} + {2}', vp_df)", dependentValue, independentValue, moderatedValue); + code.appendLine("_result = _model.fit()"); + code.appendLine("display(Markdown('### Model 2 - Dependent variable ~ Independent variable + Moderated variable'))"); + code.append("print(_result.summary())"); + if (multiCollinearity === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 2 - Multi-collinearity statistics"); + code.appendLine("from statsmodels.stats.outliers_influence import variance_inflation_factor"); + code.appendLine("_dfr = pd.DataFrame(_result.summary().tables[1].data[1:],columns=_result.summary().tables[1].data[0]).set_index('')"); + code.appendLine("for i, col in enumerate(_model.exog_names[1:]):"); + code.appendLine(" _vif = variance_inflation_factor(_model.exog, i+1)"); + code.appendLine(" _dfr.loc[col,'Tolerance'] = 1/_vif"); + code.appendLine(" _dfr.loc[col,'VIF'] = _vif"); + code.append("display(_dfr)"); + } + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 3 - Dependent variable ~ Independent variable + Moderated variable +Independent:Moderated"); + code.appendLine("import statsmodels.formula.api as smf"); + code.appendFormatLine("_model = smf.ols('{0} ~ {1} + {2} + {3}:{4}', vp_df)", dependentValue, independentValue, moderatedValue, independentValue, moderatedValue); + code.appendLine("_result = _model.fit()"); + code.appendLine("display(Markdown('### Model 3 - Dependent variable ~ Independent variable + Moderated variable +Independent:Moderated'))"); + code.append("print(_result.summary())"); + if (multiCollinearity === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 3 - Multi-collinearity statistics"); + code.appendLine("from statsmodels.stats.outliers_influence import variance_inflation_factor"); + code.appendLine("_dfr = pd.DataFrame(_result.summary().tables[1].data[1:],columns=_result.summary().tables[1].data[0]).set_index('')"); + code.appendLine("for i, col in enumerate(_model.exog_names[1:]):"); + code.appendLine(" _vif = variance_inflation_factor(_model.exog, i+1)"); + code.appendLine(" _dfr.loc[col,'Tolerance'] = 1/_vif"); + code.appendLine(" _dfr.loc[col,'VIF'] = _vif"); + code.append("display(_dfr)"); + } + // set last model number + lastModelNum = 3; + break; + case 'mediated': + // 5. Mediated + if (sobelTest === true) { + this.addCheckModules('stats'); + this.addCheckModules('vp_sobel'); + } + code.appendLine(); + code.appendLine("# Model 1 - Mediated variable ~ Independent variable"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("import statsmodels.formula.api as smf"); + code.appendFormatLine("_model = smf.ols('{0} ~ {1}', vp_df)", mediatedValue, independentValue); + code.appendLine("_result = _model.fit()"); + code.appendLine("display(Markdown('### Model 1 - Mediated variable ~ Independent variable'))"); + code.append("print(_result.summary())"); + if (multiCollinearity === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 1 - Multi-collinearity statistics"); + code.appendLine("from statsmodels.stats.outliers_influence import variance_inflation_factor"); + code.appendLine("_dfr = pd.DataFrame(_result.summary().tables[1].data[1:],columns=_result.summary().tables[1].data[0]).set_index('')"); + code.appendLine("for i, col in enumerate(_model.exog_names[1:]):"); + code.appendLine(" _vif = variance_inflation_factor(_model.exog, i+1)"); + code.appendLine(" _dfr.loc[col,'Tolerance'] = 1/_vif"); + code.appendLine(" _dfr.loc[col,'VIF'] = _vif"); + code.append("display(_dfr)"); + } + if (sobelTest === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 1 - Sobel test"); + code.appendFormatLine("_sobel_M1 = _result.params[{0}]", independent); + code.appendFormat("_sobel_M1se = _result.bse[{0}]", independent); + } + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 2 - Dependent variable ~ Independent variable"); + code.appendLine("import statsmodels.formula.api as smf"); + code.appendFormatLine("_model = smf.ols('{0} ~ {1}', vp_df)", dependentValue, independentValue); + code.appendLine("_result = _model.fit()"); + code.appendLine("display(Markdown('### Model 2 - Dependent variable ~ Independent variable'))"); + code.append("print(_result.summary())"); + if (multiCollinearity === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 2 - Multi-collinearity statistics"); + code.appendLine("from statsmodels.stats.outliers_influence import variance_inflation_factor"); + code.appendLine("_dfr = pd.DataFrame(_result.summary().tables[1].data[1:],columns=_result.summary().tables[1].data[0]).set_index('')"); + code.appendLine("for i, col in enumerate(_model.exog_names[1:]):"); + code.appendLine(" _vif = variance_inflation_factor(_model.exog, i+1)"); + code.appendLine(" _dfr.loc[col,'Tolerance'] = 1/_vif"); + code.appendLine(" _dfr.loc[col,'VIF'] = _vif"); + code.append("display(_dfr)"); + } + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 3 - Dependent variable ~ Independent variable + Mediated variable"); + code.appendLine("import statsmodels.formula.api as smf"); + code.appendFormatLine("_model = smf.ols('{0} ~ {1} + {2}', vp_df)", dependentValue, independentValue, mediatedValue); + code.appendLine("_result = _model.fit()"); + code.appendLine("display(Markdown('### Model 3 - Dependent variable ~ Independent variable + Mediated variable'))"); + code.append("print(_result.summary())"); + if (multiCollinearity === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 3 - Multi-collinearity statistics"); + code.appendLine("from statsmodels.stats.outliers_influence import variance_inflation_factor"); + code.appendLine("_dfr = pd.DataFrame(_result.summary().tables[1].data[1:],columns=_result.summary().tables[1].data[0]).set_index('')"); + code.appendLine("for i, col in enumerate(_model.exog_names[1:]):"); + code.appendLine(" _vif = variance_inflation_factor(_model.exog, i+1)"); + code.appendLine(" _dfr.loc[col,'Tolerance'] = 1/_vif"); + code.appendLine(" _dfr.loc[col,'VIF'] = _vif"); + code.append("display(_dfr)"); + } + if (sobelTest === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 3 - Sobel test"); + code.appendFormatLine("_sobel_M3 = _result.params[{0}]", mediated); + code.appendFormatLine("_sobel_M3se = _result.bse[{0}]", mediated); + code.appendLine(); + code.appendLine("# Mediated linear regression: Sobel test"); + code.appendLine("from scipy import stats"); + code.appendLine("_res = vp_sobel(_sobel_M1, _sobel_M3, _sobel_M1se, _sobel_M3se)"); + code.appendLine("display(Markdown('### Sobel test'))"); + code.append("display(pd.DataFrame(data={'Sobel Z-score':_res[0],'p-value':_res[2]},index=['Sobel test']))"); + } + // set last model number + lastModelNum = 3; + break; + case 'dummy': + // 6. Dummy variable + code.appendLine(); + code.appendLine("# Dummy variable linear regression"); + code.appendLine("import statsmodels.formula.api as smf"); + code.appendFormatLine("_model = smf.ols('{0} ~ {1}', vp_df)" + , dependentValue, independentMulti.map(item => { + let checked = $(that.wrapSelector('.vp-categorical-box input[data-name="' + item.name + '"]')).prop('checked'); + if (checked === true) { + return 'C(' + item.name + ')'; + } else { + return item.name; + } + }).join(' + ')); + code.appendLine("_result = _model.fit()"); + code.append("print(_result.summary())"); + if (multiCollinearity === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Multi-collinearity statistics"); + code.appendLine("from IPython.display import display"); + code.appendLine("from statsmodels.stats.outliers_influence import variance_inflation_factor"); + code.appendLine("_dfr = pd.DataFrame(_result.summary().tables[1].data[1:],columns=_result.summary().tables[1].data[0]).set_index('')"); + code.appendLine("for i, col in enumerate(_model.exog_names[1:]):"); + code.appendLine(" _vif = variance_inflation_factor(_model.exog, i+1)"); + code.appendLine(" _dfr.loc[col,'Tolerance'] = 1/_vif"); + code.appendLine(" _dfr.loc[col,'VIF'] = _vif"); + code.append("display(_dfr)"); + } + break; + } + + // Residual option + if (statistics === true || normTest === true || histogram === true || scatterplot === true || rmse === true) { + let residualTitle = 'Residual' + if (lastModelNum > 0) { + residualTitle += ' - Model ' + lastModelNum; + } + code.appendLine(); + code.appendLine(); + code.appendFormatLine("# {0}", residualTitle); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("from scipy import stats"); + code.appendLine("import statsmodels.api as sm"); + if (testType === 'multiple' && ['stepwise', 'backward', 'forward'].includes(method)) { + code.appendLine("_predict = _result.predict(sm.add_constant(vp_df[_model.exog_names[1:]]))"); + } else { + code.appendLine("_predict = _result.predict(vp_df)"); + } + + code.appendLine("_residual = _result.resid"); + code.appendLine("vp_residual = pd.DataFrame({'predict':_predict,'residual':_residual,"); + code.appendLine(" 'predict_z':stats.zscore(_predict),'residual_z':stats.zscore(_residual)})"); + code.appendFormatLine("display(Markdown('### {0}'))", residualTitle); + code.append("display(vp_residual)"); + + if (statistics === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Residual statistics"); + code.appendLine("display(Markdown('### Residual statistics'))"); + code.appendLine("display(pd.DataFrame(data={'Min':vp_residual.min(),'Max':vp_residual.max(),'Mean':vp_residual.mean(numeric_only=True),"); + code.append(" 'Std. Deviation':vp_residual.std(numeric_only=True),'N':vp_residual.count()}))"); + } + if (normTest === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Resisual Normality test (Shapiro-Wilk)"); + code.appendLine("_res = stats.shapiro(vp_residual['residual_z'])"); + code.appendLine("display(Markdown('### Residual Normality test (Shapiro-Wilk)'))"); + code.append("display(pd.DataFrame(data={'Statistic':_res.statistic,'p-value':_res.pvalue},index=['Resisual Normality test (Shapiro-Wilk)']))"); + } + if (histogram === true || scatterplot === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("import seaborn as sns"); + code.appendLine("import warnings"); + code.append("with warnings.catch_warnings():"); + let displayNum = 1; + if (histogram === true) { + code.appendLine(); + code.appendLine(); + code.appendLine(" # Residual histogram"); + code.appendFormatLine(" plt.subplot(2,2,{0})", displayNum++); + code.appendLine(" warnings.simplefilter(action='ignore', category=Warning)"); + code.appendLine(" sns.histplot(data=vp_residual, x='residual_z', kde=True)"); + code.appendLine(" plt.title(f'Dependent variable: {_model.endog_names}')"); + code.append(" plt.xlabel('Regression Standardized residual')"); + } + if (scatterplot === true) { + code.appendLine(); + code.appendLine(); + code.appendLine(" # Residual scatterplot"); + code.appendFormatLine(" plt.subplot(2,2,{0})", displayNum++); + code.appendLine(" sns.scatterplot(data=vp_residual, x='predict_z', y='residual_z')"); + code.appendLine(" plt.title(f'Dependent variable: {_model.endog_names}')"); + code.appendLine(" plt.xlabel('Regression Standardized predicted value')"); + code.append(" plt.ylabel('Regression Standardized residual')"); + } + code.appendLine(); + code.appendLine(); + code.appendLine(" plt.tight_layout()"); + code.append(" plt.show()"); + } + if (rmse === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# RMSE (Root Mean Squared Error)"); + code.appendLine("_rmse = np.sqrt(_result.mse_resid)"); + code.appendLine("display(Markdown('### RMSE (Root Mean Squared Error)'))"); + code.append("display(Markdown(f'RMSE: {_rmse}'))"); + } + } + + codeList.push(code.toString()); + return codeList; + } + + } + + return Regression; +}); \ No newline at end of file diff --git a/visualpython/js/m_stats/ReliabAnalysis.js b/visualpython/js/m_stats/ReliabAnalysis.js new file mode 100644 index 00000000..989931b0 --- /dev/null +++ b/visualpython/js/m_stats/ReliabAnalysis.js @@ -0,0 +1,158 @@ +/* + * Project Name : Visual Python + * Description : GUI-based Python code generator + * File Name : ReliabAnalysis.js + * Author : Black Logic + * Note : Reliability Analysis + * License : GNU GPLv3 with Visual Python special exception + * Date : 2023. 05. 24 + * Change Date : + */ + +//============================================================================ +// [CLASS] ReliabAnalysis +//============================================================================ +define([ + __VP_TEXT_LOADER__('vp_base/html/m_stats/reliabAnalysis.html'), + 'vp_base/js/com/com_util', + 'vp_base/js/com/com_Const', + 'vp_base/js/com/com_String', + 'vp_base/js/com/component/PopupComponent', + 'vp_base/js/com/component/DataSelector', + 'vp_base/js/com/component/MultiSelector', + 'vp_base/js/m_apps/Subset' +], function(eqHTML, com_util, com_Const, com_String, PopupComponent, DataSelector, MultiSelector, Subset) { + + /** + * ReliabAnalysis + */ + class ReliabAnalysis extends PopupComponent { + _init() { + super._init(); + /** Write codes executed before rendering */ + this.config.sizeLevel = 2; + this.config.checkModules = ['pd', 'np', 'vp_cronbach_alpha']; + + this.state = { + data: '', + variable: [], + ...this.state + }; + + this.subsetEditor = null; + this.columnSelector = null; + } + + _bindEvent() { + super._bindEvent(); + /** Implement binding events */ + var that = this; + + $(this.wrapSelector('#data')).on('change', function() { + let data = $(this).val(); + that.handleVariableChange(data); + }); + } + + handleVariableChange(data) { + this.state.data = data; + // render variable selector + this.columnSelector = new MultiSelector(this.wrapSelector('#variable'), + { mode: 'columns', parent: data, showDescription: false } + ); + } + + templateForBody() { + let page = $(eqHTML); + let that = this; + + // generate dataselector + let dataSelector = new DataSelector({ + pageThis: this, id: 'data', placeholder: 'Select data', required: true, boxClasses: 'vp-flex-gap5', + allowDataType: ['DataFrame'], withPopup: false, + finish: function(data, type) { + that.state.data = data; + $(that.wrapSelector('#data')).trigger('change'); + }, + select: function(data, type) { + that.state.data = data; + $(that.wrapSelector('#data')).trigger('change'); + } + }); + $(page).find('#data').replaceWith(dataSelector.toTagString()); + + return page; + } + + render() { + super.render(); + let that = this; + + // render Subset + this.subsetEditor = new Subset({ + pandasObject: '', + config: { name: 'Subset', category: this.name } }, + { + useAsModule: true, + useInputColumns: true, + targetSelector: this.wrapSelector('#data'), + pageThis: this, + finish: function(code) { + $(that.wrapSelector('#data')).val(code); + that.handleVariableChange(code); + } + }); + + // render variable selector + this.columnSelector = new MultiSelector(this.wrapSelector('#variable'), + { mode: 'columns', parent: this.state.data, selectedList: this.state.variable.map(x=>x.code), showDescription: false } + ); + } + + generateCode() { + let { data, variable } = this.state; + let codeList = []; + let code = new com_String(); + let that = this; + + // data declaration + code.appendFormat("vp_df = {0}", data); + if (this.columnSelector) { + let columns = this.columnSelector.getDataList(); + this.state.variable = columns; + if (columns.length > 0) { + code.appendFormat("[[{0}]]", columns.map(x => x.code).join(', ')); + } + } + code.appendLine('.dropna().copy()'); + + // Inner function : vp_cronbach_alpha + + // Cronbach alpha + code.appendLine(""); + code.appendLine("# Cronbach alpha"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("display(Markdown('### Cronbach alpha'))"); + code.appendLine("display(pd.DataFrame({'Cronbach alpha':vp_cronbach_alpha(vp_df), 'N':vp_df.shape[1]},index=['Reliability statistics']).round(3))"); + code.appendLine(""); + + // Item-total Statistics + code.appendLine("# Item-Total Statistics"); + code.appendLine("_dfr = pd.DataFrame()"); + code.appendLine("for i, col in enumerate(vp_df.columns):"); + code.appendLine(" _sr = vp_df.drop(col,axis=1).sum(axis=1)"); + code.appendLine(" _df_t = pd.DataFrame(data={'Scale Mean if Item Deleted':_sr.mean(),'Scale Variance if Item Deleted':_sr.var(),"); + code.appendLine(" 'Corrected Item-Total Correlation':_sr.corr(vp_df[col]),"); + code.appendLine(" 'Cronbach Alpha if Item Deleted':vp_cronbach_alpha(vp_df.drop(col,axis=1))}, index=[col])"); + code.appendLine(" _dfr = pd.concat([_dfr, _df_t])"); + code.appendLine("display(Markdown('### Item-Total Statistics'))"); + code.append("display(_dfr.round(3))"); + codeList.push(code.toString()); + + return codeList; + } + + } + + return ReliabAnalysis; +}); \ No newline at end of file diff --git a/visualpython/js/m_stats/StudentstTest.js b/visualpython/js/m_stats/StudentstTest.js new file mode 100644 index 00000000..d9b5b1a4 --- /dev/null +++ b/visualpython/js/m_stats/StudentstTest.js @@ -0,0 +1,410 @@ +/* + * Project Name : Visual Python + * Description : GUI-based Python code generator + * File Name : StudentstTest.js + * Author : Black Logic + * Note : Student's t-test + * License : GNU GPLv3 with Visual Python special exception + * Date : 2023. 05. 09 + * Change Date : + */ + +//============================================================================ +// [CLASS] StudentstTest +//============================================================================ +define([ + __VP_TEXT_LOADER__('vp_base/html/m_stats/studentstTest.html'), + 'vp_base/js/com/com_util', + 'vp_base/js/com/com_Const', + 'vp_base/js/com/com_String', + 'vp_base/js/com/com_generatorV2', + 'vp_base/js/com/component/PopupComponent', + 'vp_base/js/com/component/DataSelector', + 'vp_base/js/com/component/SuggestInput', + 'vp_base/js/m_apps/Subset' +], function(stHTML, com_util, com_Const, com_String, com_generator, PopupComponent, DataSelector, SuggestInput, Subset) { + + /** + * StudentstTest + * - confidence_interval is available on upper 1.10.0 version of scipy + */ + class StudentstTest extends PopupComponent { + _init() { + super._init(); + /** Write codes executed before rendering */ + this.config.sizeLevel = 2; + this.config.checkModules = ['pd']; + this.config.docs = 'https://docs.scipy.org/doc/scipy/reference/'; + + this.state = { + testType: 'one-sample', + inputType: 'long-data', + data: '', + dataType: '', + testVariable: '', + testVariable1: '', + testVariable2: '', + groupingVariable: '', + group1: '', + group2: '', + group1_istext: true, + group2_istext: true, + pairedVariable1: '', + pairedVariable2: '', + testValue: '', + alterHypo: 'two-sided', + confInt: '95', + ...this.state + }; + + this.columnBindDict = { + 'one-sample': ['testVariable'], + 'two-sample': ['testVariable', 'testVariable1', 'testVariable2', 'groupingVariable'], + 'paired-sample': ['pairedVariable1', 'pairedVariable2'] + }; + + this.subsetEditor = null; + } + + _unbindEvent() { + super._unbindEvent(); + var that = this; + + $(document).off(this.wrapSelector('#testVariable'), 'change'); + $(document).off(this.wrapSelector('#groupingVariable'), 'change'); + } + + _bindEvent() { + super._bindEvent(); + /** Implement binding events */ + var that = this; + + // change test type + $(this.wrapSelector('#testType')).on('change', function() { + let testType = $(this).val(); + that.state.testType = testType; + + that.handleVariableChange(that.state.data); + + $(that.wrapSelector('.vp-st-option')).hide(); + $(that.wrapSelector('.vp-st-option.' + testType)).show(); + if (testType === 'two-sample') { + $(that.wrapSelector('.vp-st-option.two-sample-' + that.state.inputType)).show(); + } + }); + + // change input type + $(this.wrapSelector('input[name="inputType"]:radio')).on('change', function() { + let inputType = $(this).val(); + that.state.inputType = inputType; + $(that.wrapSelector('.vp-st-option.two-sample-long-data')).hide(); + $(that.wrapSelector('.vp-st-option.two-sample-wide-data')).hide(); + $(that.wrapSelector('.vp-st-option.two-sample-' + inputType)).show(); + }); + + // data change event + $(this.wrapSelector('#data')).on('change', function() { + let data = $(this).val(); + that.handleVariableChange(data); + }); + + // change test variable + $(document).on('change', this.wrapSelector('#testVariable'), function() { + if (that.state.testType === 'one-sample') { + // get mean of data and show on placeholder + $(that.wrapSelector('#testValue')).prop('placeholder', ''); + vpKernel.execute(com_util.formatString("int({0}[{1}].mean())", that.state.data, that.state.testVariable)).then(function(resultObj) { + let { result } = resultObj; + $(that.wrapSelector('#testValue')).prop('placeholder', result); + }); + } + }); + + // change grouping variable + $(document).on('change', this.wrapSelector('#groupingVariable'), function() { + let colCode = $(this).val(); + var colName = $(this).find('option:selected').text(); + var colDtype = $(this).find('option:selected').attr('data-type'); + that.state.groupingVariable = colCode; + // get result and load column list + vpKernel.getColumnCategory(that.state.data, colCode).then(function(resultObj) { + let { result } = resultObj; + $(that.wrapSelector('#group1')).val(''); + $(that.wrapSelector('#group2')).val(''); + that.state.group1 = ''; + that.state.group2 = ''; + that.state.group1_istext = true; + that.state.group2_istext = true; + try { + var category = JSON.parse(result); + // if (category && category.length > 0 && colDtype == 'object') { + // // if it's categorical column and its dtype is object, check 'Text' as default + // category.forEach(obj => { + // let selected1 = obj.value === that.state.group1; + // let selected2 = obj.value === that.state.group2; + // $(that.wrapSelector('#group1')).append(``); + // $(that.wrapSelector('#group2')).append(``); + // }); + // } + var groupSuggest1 = new SuggestInput(); + groupSuggest1.setComponentID('group1'); + groupSuggest1.addClass('vp-input vp-state'); + groupSuggest1.setSuggestList(function() { return category; }); + groupSuggest1.setNormalFilter(true); + groupSuggest1.setPlaceholder('Select value'); + $(that.wrapSelector('#group1')).replaceWith(groupSuggest1.toTagString()); + var groupSuggest2 = new SuggestInput(); + groupSuggest2.setComponentID('group2'); + groupSuggest2.addClass('vp-input vp-state'); + groupSuggest2.setSuggestList(function() { return category; }); + groupSuggest2.setNormalFilter(true); + groupSuggest2.setPlaceholder('Select value'); + $(that.wrapSelector('#group2')).replaceWith(groupSuggest2.toTagString()); + + if (category && category.length > 0) { + that.state.group1 = category[0].value; + that.state.group2 = category[0].value; + } + + if (colDtype == 'object') { + // check as default + $(that.wrapSelector('#group1_istext')).prop('checked', true); + $(that.wrapSelector('#group2_istext')).prop('checked', true); + that.state.group1_istext = true; + that.state.group2_istext = true; + } else { + $(that.wrapSelector('#group1_istext')).prop('checked', false); + $(that.wrapSelector('#group2_istext')).prop('checked', false); + that.state.group1_istext = false; + that.state.group2_istext = false; + } + } catch { + $(that.wrapSelector('#group1')).val(''); + $(that.wrapSelector('#group2')).val(''); + } + }); + }); + } + + handleVariableChange(data) { + let that = this; + this.state.data = data; + let columnBindList = this.columnBindDict[this.state.testType]; + if (this.state.dataType === 'DataFrame') { + // DataFrame + columnBindList.forEach(col => { + $(that.wrapSelector('#' + col)).prop('disabled', false); + }); + com_generator.vp_bindColumnSource(that, 'data', columnBindList, 'select', false, false); + } else { + // Series + columnBindList.forEach(col => { + $(that.wrapSelector('#' + col)).html(''); + $(that.wrapSelector('#' + col)).prop('disabled', true); + }); + } + } + + templateForBody() { + let page = $(stHTML); + let that = this; + + // generate dataselector + let dataSelector = new DataSelector({ + pageThis: this, id: 'data', placeholder: 'Select data', required: true, boxClasses: 'vp-flex-gap5', + allowDataType: ['DataFrame'], withPopup: false, + finish: function(data, type) { + that.state.data = data; + that.state.dataType = type; + $(that.wrapSelector('#data')).trigger('change'); + }, + select: function(data, type) { + that.state.data = data; + that.state.dataType = type; + $(that.wrapSelector('#data')).trigger('change'); + } + }); + $(page).find('#data').replaceWith(dataSelector.toTagString()); + + return page; + } + + render() { + super.render(); + let that = this; + + // render Subset + this.subsetEditor = new Subset({ + pandasObject: '', + config: { name: 'Subset' } }, + { + useAsModule: true, + useInputColumns: true, + targetSelector: this.wrapSelector('#data'), + pageThis: this, + finish: function(code, state) { + that.state.data = code; + that.state.dataType = state.returnType; + $(that.wrapSelector('#data')).val(code); + $(that.wrapSelector('#data')).trigger('change'); + } + }); + + if (this.state.data !== '') { + let columnBindList = this.columnBindDict[this.state.testType]; + com_generator.vp_bindColumnSource(this, 'data', columnBindList, 'select', false, false); + } + + // control display option + $(this.wrapSelector('.vp-st-option')).hide(); + $(this.wrapSelector('.vp-st-option.' + this.state.testType)).show(); + if (this.state.testType === 'two-sample') { + $(this.wrapSelector('.vp-st-option.two-sample-' + this.state.inputType)).show(); + } + } + + generateCode() { + let { + testType, inputType, data, + testVariable, testVariable1, testVariable2, groupingVariable, + pairedVariable1, pairedVariable2, + group1, group2, group1_istext, group2_istext, + testValue, alterHypo, confInt + } = this.state; + let codeList = []; + let code = new com_String(); + + // 95% -> 0.95 + confInt = confInt/100; + + switch (testType) { + case 'one-sample': + code.appendLine("# One-sample t-test"); + // variable declaration + code.appendFormatLine("vp_df = {0}.dropna().copy()", data); + code.appendLine(""); + // 1. Normality test + code.appendLine("# Normality test (Shapiro-Wilk)"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("from scipy import stats"); + code.appendFormatLine("_res = stats.shapiro(vp_df[{0}])", testVariable); + code.appendLine("display(Markdown('### Normality test (Shapiro-Wilk)'))"); + code.appendLine("display(pd.DataFrame(data={'Statistic':_res.statistic,'p-value':_res.pvalue},index=['Normality test (Shapiro-Wilk)']))"); + code.appendLine(""); + // 2. One-sample Statistics + code.appendLine("# Statistics"); + code.appendLine("display(Markdown('### Statistics'))"); + code.appendFormatLine("display(pd.DataFrame(data={'N':vp_df[{0}].size,'Mean':vp_df[{1}].mean(),", testVariable, testVariable); + code.appendFormatLine(" 'Std. Deviation':vp_df[{0}].std(),", testVariable); + code.appendFormatLine(" 'Std. Error Mean':vp_df[{0}].std()/np.sqrt(vp_df[{1}].size)},", testVariable, testVariable); + code.appendLine(" index=['Statistics']))"); + code.appendLine(""); + // 3. One-sample t-test + code.appendLine("# One-sample t-test"); + code.appendFormatLine("_res = stats.ttest_1samp(vp_df[{0}], popmean={1}, alternative='{2}')", testVariable, testValue, alterHypo); + code.appendFormatLine("_lower, _upper = _res.confidence_interval(confidence_level={0})", confInt); + code.appendLine("display(Markdown('### One-sample t-test'))"); + code.appendFormatLine("display(pd.DataFrame(data={'Statistic':_res.statistic,'dof':_res.df,'Alternative':'{0}',", alterHypo); + code.appendFormatLine(" 'p-value':_res.pvalue,'Test Value':{0},'Mean difference':vp_df[{1}].mean()-{2},", testValue, testVariable, testValue); + code.appendFormatLine(" 'Confidence interval':{0},'Lower':_lower,'Upper':_upper},", confInt); + code.append(" index=['One-sample t-test']))"); + break; + case 'two-sample': + code.appendLine("# Independent two-sample t-test"); + // variable declaration + if (inputType === 'long-data') { + code.appendFormatLine("vp_df1 = {0}[({1}[{2}] == {3})][{4}].dropna().copy()", data, data, groupingVariable, com_util.convertToStr(group1, group1_istext), testVariable); + code.appendFormatLine("vp_df2 = {0}[({1}[{2}] == {3})][{4}].dropna().copy()", data, data, groupingVariable, com_util.convertToStr(group2, group2_istext), testVariable); + } else if (inputType === 'wide-data') { + code.appendFormatLine("vp_df1 = {0}[{1}].dropna().copy()", data, testVariable1); + code.appendFormatLine("vp_df2 = {0}[{1}].dropna().copy()", data, testVariable2); + } + code.appendLine(""); + // 1. Normality test + code.appendLine("# Normality test (Shapiro-Wilk)"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("from scipy import stats"); + code.appendLine("_res1 = stats.shapiro(vp_df1)"); + code.appendLine("_res2 = stats.shapiro(vp_df2)"); + code.appendLine("display(Markdown('### Normality test (Shapiro-Wilk)'))"); + code.appendLine("display(pd.DataFrame(data={'Statistic':[_res1.statistic,_res2.statistic],'p-value':[_res1.pvalue,_res2.pvalue]},"); + code.appendLine(" index=[['Normality test (Shapiro-Wilk)' for i in range(2)],['Variable1','Variable2']]))"); + code.appendLine(""); + // 2. Equal Variance test + code.appendLine("# Equal Variance test (Levene)"); + code.appendLine("display(Markdown('### Equal Variance test (Levene)'))"); + code.appendLine("_res = stats.levene(vp_df1, vp_df2, center='mean')"); + code.appendLine("display(pd.DataFrame(data={'Statistic':_res.statistic,'p-value':_res.pvalue}, index=['Equal Variance test (Levene)']))"); + code.appendLine(""); + // 3. Independent two-sample Statistics + code.appendLine("# Statistics"); + code.appendLine("display(Markdown('### Statistics'))"); + code.appendLine("display(pd.DataFrame(data={'N':[vp_df1.size,vp_df2.size],"); + code.appendLine(" 'Mean':[vp_df1.mean(),vp_df2.mean()],"); + code.appendLine(" 'Std. Deviation':[vp_df1.std(),vp_df2.std()],"); + code.appendLine(" 'Std. Error mean':[vp_df1.std()/np.sqrt(vp_df1.size),"); + code.appendLine(" vp_df2.std()/np.sqrt(vp_df2.size )]},"); + code.appendLine(" index=[['Statistics' for i in range(2)],['Variable1','Variable2']]))"); + code.appendLine(""); + // 4. Independent two-sample t-test + code.appendLine("# Independent two-sample t-test"); + code.appendFormatLine("_res1 = stats.ttest_ind(vp_df1, vp_df2, equal_var=True, alternative='{0}')", alterHypo); + code.appendFormatLine("_res2 = stats.ttest_ind(vp_df1, vp_df2, equal_var=False, alternative='{0}')", alterHypo); + code.appendLine("display(Markdown('### Independent two-sample t-test'))"); + code.appendFormatLine("display(pd.DataFrame(data={'Statistic':[_res1.statistic,_res2.statistic],'Alternative':['{0}' for i in range(2)],", alterHypo); + code.appendLine(" 'p-value':[_res1.pvalue,_res2.pvalue],"); + code.appendLine(" 'Mean difference':[vp_df1.mean()-vp_df2.mean() for i in range(2)]},"); + code.appendLine(" index=[['Independent two-sample t-test' for i in range(2)],['Equal variance' for i in range(2)],[True,False]]))"); + code.append("display(Markdown('If equal_var is False, perform Welch\\\'s t-test, which does not assume equal population variance'))"); + break; + case 'paired-sample': + // variable declaration + code.appendLine("# Paired samples t-test"); + code.appendFormatLine("vp_df = {0}.dropna().copy()", data); + code.appendLine(""); + code.appendFormatLine("try: vp_df[{0}].reset_index(drop=True, inplace=True)", pairedVariable1); + code.appendLine("except: pass"); + code.appendFormatLine("try: vp_df[{0}].reset_index(drop=True, inplace=True)", pairedVariable2); + code.appendLine("except: pass"); + code.appendLine(""); + // 1. Normality test + code.appendLine("# Normality test (Shapiro-Wilk)"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("from scipy import stats"); + code.appendFormatLine("_res = stats.shapiro(vp_df[{0}]-vp_df[{1}])", pairedVariable1, pairedVariable2); + code.appendLine("display(Markdown('### Normality test (Shapiro-Wilk)'))"); + code.appendLine("display(pd.DataFrame(data={'Statistic':_res.statistic,'p-value':_res.pvalue},"); + code.appendLine(" index=['Normality test (Shapiro-Wilk): Paired differences']))"); + code.appendLine(""); + // 2. Paired samples Statistics + code.appendLine("# Statistics"); + code.appendLine("display(Markdown('### Statistics'))"); + code.appendFormatLine("display(pd.DataFrame(data={'N':[vp_df[{0}].size,vp_df[{1}].size,vp_df[{2}].size],", pairedVariable1, pairedVariable2, pairedVariable1); + code.appendFormatLine(" 'Mean':[vp_df[{0}].mean(),vp_df[{1}].mean(),(vp_df[{2}]-vp_df[{3}]).mean()],", pairedVariable1, pairedVariable2, pairedVariable1, pairedVariable2); + code.appendFormatLine(" 'Std. Deviation':[vp_df[{0}].std(),vp_df[{1}].std(),(vp_df[{2}]-vp_df[{3}]).std()],", pairedVariable1, pairedVariable2, pairedVariable1, pairedVariable2); + code.appendFormatLine(" 'Std. Error mean':[vp_df[{0}].std()/np.sqrt(vp_df[{1}].size),", pairedVariable1, pairedVariable1); + code.appendFormatLine(" vp_df[{0}].std()/np.sqrt(vp_df[{1}].size),", pairedVariable2, pairedVariable2); + code.appendFormatLine(" (vp_df[{0}]-vp_df[{1}]).std()/np.sqrt(vp_df[{2}].size)]},", pairedVariable1, pairedVariable2, pairedVariable1); + code.appendLine(" index=[['Statistics' for i in range(3)],['Variable1','Variable2','Paired differences']]))"); + code.appendLine(""); + // 3. Paired samples t-test + code.appendLine("# Paired samples t-test"); + code.appendFormatLine("_res = stats.ttest_rel(vp_df[{0}], vp_df[{1}], alternative='{2}')", pairedVariable1, pairedVariable2, alterHypo); + code.appendFormatLine("_lower, _upper = _res.confidence_interval(confidence_level={0})", confInt); + code.appendLine("display(Markdown('### Paired samples t-test'))"); + code.appendFormatLine("display(pd.DataFrame(data={'Statistic':_res.statistic,'dof':_res.df,'Alternative':'{0}',", alterHypo); + code.appendFormatLine(" 'p-value':_res.pvalue,'Mean difference':(vp_df[{0}]-vp_df[{1}]).mean(),", pairedVariable1, pairedVariable2); + code.appendFormatLine(" 'Confidence interval':{0},'Lower':_lower,'Upper':_upper},", confInt); + code.append(" index=['Paired samples t-test']))"); + break; + } + codeList.push(code.toString()); + + + return codeList; + } + + } + + return StudentstTest; +}); \ No newline at end of file diff --git a/visualpython/js/m_stats/__init__.py b/visualpython/js/m_stats/__init__.py new file mode 100644 index 00000000..1e8c9919 --- /dev/null +++ b/visualpython/js/m_stats/__init__.py @@ -0,0 +1 @@ +print('Visual Python') diff --git a/visualpython/js/m_visualize/Chart.js b/visualpython/js/m_visualize/Chart.js index 4a3293c9..d979a6e1 100644 --- a/visualpython/js/m_visualize/Chart.js +++ b/visualpython/js/m_visualize/Chart.js @@ -18,12 +18,13 @@ define([ 'vp_base/js/com/com_String', 'vp_base/js/com/com_Const', 'vp_base/js/com/com_util', + 'vp_base/js/com/com_interface', 'vp_base/js/com/com_generator', 'vp_base/js/com/component/PopupComponent', 'vp_base/js/com/component/FileNavigation', 'vp_base/js/com/component/SuggestInput', 'vp_base/js/com/component/DataSelector' -], function(chartHTml, chartCss, com_String, com_Const, com_util, com_generator, PopupComponent, FileNavigation, SuggestInput, DataSelector) { +], function(chartHTml, chartCss, com_String, com_Const, com_util, com_interface, com_generator, PopupComponent, FileNavigation, SuggestInput, DataSelector) { /** * Chart @@ -35,6 +36,7 @@ define([ this.config.dataview = false; this.config.sizeLevel = 2; this.config.checkModules = ['plt']; + this.config.docs = 'https://matplotlib.org/stable/'; this.setDefaultVariables(); this.state = { @@ -267,14 +269,16 @@ define([ $(this.wrapSelector('#vp_plImportRun')).click(function() { // generateImportCode var code = that.generateImportCode(); + // DEPRECATED: no longer save to block as default // create block and run it - $('#vp_wrapper').trigger({ - type: 'create_option_page', - blockType: 'block', - menuId: 'lgExe_code', - menuState: { taskState: { code: code } }, - afterAction: 'run' - }); + // $('#vp_wrapper').trigger({ + // type: 'create_option_page', + // blockType: 'block', + // menuId: 'lgExe_code', + // menuState: { taskState: { code: code } }, + // afterAction: 'run' + // }); + com_interface.insertCell('code', code, true, 'Visualization > Chart'); }); } @@ -1308,8 +1312,8 @@ define([ 'numerical': [ { label: 'count', method: 'count()' }, { label: 'unique count', method: 'unique()' }, - { label: 'sum', method: 'sum()' }, - { label: 'average', method: 'mean()' }, + { label: 'sum', method: 'sum(numeric_only=True)' }, + { label: 'average', method: 'mean(numeric_only=True)' }, { label: 'min', method: 'min()' }, { label: 'max', method: 'max()' } ] diff --git a/visualpython/js/m_visualize/Plotly.js b/visualpython/js/m_visualize/Plotly.js index e77db277..684840b3 100644 --- a/visualpython/js/m_visualize/Plotly.js +++ b/visualpython/js/m_visualize/Plotly.js @@ -34,6 +34,7 @@ define([ this.config.importButton = true; this.config.dataview = false; this.config.checkModules = ['px']; + this.config.docs = 'https://plotly.com/python-api-reference/index.html'; this.state = { chartType: 'scatter', @@ -450,20 +451,20 @@ define([ } } else { var errorContent = ''; - if (msg.content.ename) { + if (msg.content && msg.content.ename) { errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue); } $(that.wrapSelector('#vp_ptPreview')).html(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); } }).catch(function(resultObj) { let { msg } = resultObj; var errorContent = ''; - if (msg.content.ename) { + if (msg.content && msg.content.ename) { errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue); } $(that.wrapSelector('#vp_ptPreview')).html(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); }); }); } diff --git a/visualpython/js/m_visualize/Seaborn.js b/visualpython/js/m_visualize/Seaborn.js index 555d29c7..bea6cfd1 100644 --- a/visualpython/js/m_visualize/Seaborn.js +++ b/visualpython/js/m_visualize/Seaborn.js @@ -18,12 +18,13 @@ define([ 'vp_base/js/com/com_String', 'vp_base/js/com/com_generatorV2', 'vp_base/js/com/com_util', + 'vp_base/js/com/com_interface', 'vp_base/js/com/component/PopupComponent', 'vp_base/js/com/component/SuggestInput', 'vp_base/js/com/component/VarSelector2', 'vp_base/data/m_visualize/seabornLibrary', 'vp_base/js/com/component/DataSelector' -], function(chartHTml, chartCss, com_String, com_generator, com_util, PopupComponent, SuggestInput, VarSelector2, SEABORN_LIBRARIES, DataSelector) { +], function(chartHTml, chartCss, com_String, com_generator, com_util, com_interface, PopupComponent, SuggestInput, VarSelector2, SEABORN_LIBRARIES, DataSelector) { class Seaborn extends PopupComponent { _init() { @@ -31,7 +32,9 @@ define([ this.config.dataview = false; this.config.size = { width: 1064, height: 550 }; + this.config.autoScroll = false; this.config.checkModules = ['plt', 'sns']; + this.config.docs = 'https://seaborn.pydata.org/index.html'; this.state = { chartType: 'scatterplot', @@ -51,6 +54,7 @@ define([ stat: '', showValues: false, showValuesPrecision: '', + errorbar: '', sortBy: 'y', sortType: '', sortHue: '', @@ -92,8 +96,8 @@ define([ this.chartTypeList = { 'Relational': [ 'scatterplot', 'lineplot' ], 'Distributions': [ 'histplot', 'kdeplot', 'rugplot' ], - 'Categorical': [ 'stripplot', 'swarmplot', 'boxplot', 'violinplot', 'pointplot', 'barplot' ], - 'ETC': [ 'countplot' ] + 'Categorical': [ 'stripplot', 'swarmplot', 'boxplot', 'violinplot', 'pointplot', 'barplot', 'countplot' ], + 'Axes-level': [ 'heatmap' ], } this.legendPosList = [ @@ -198,7 +202,9 @@ define([ $(that.wrapSelector('#kde')).closest('.sb-option').show(); $(that.wrapSelector('#stat')).closest('.sb-option').show(); } else if (chartType == 'barplot') { + $(that.wrapSelector('#orient')).closest('.sb-option').show(); $(that.wrapSelector('#showValues')).closest('.sb-option').show(); + $(that.wrapSelector('#errorbar')).closest('.sb-option').show(); if (that.state.setXY === false) { if (that.state.x !== '' && that.state.y !== '') { $(that.wrapSelector('#sortBy')).closest('.sb-option').show(); @@ -217,6 +223,10 @@ define([ $(that.wrapSelector('#sortHue')).closest('.sb-option').show(); } } + } else if (chartType == 'heatmap') { + $(that.wrapSelector('#annot')).closest('.sb-option').show(); + } else if (chartType === 'lineplot') { + $(that.wrapSelector('#errorbar')).closest('.sb-option').show(); } }); @@ -358,29 +368,33 @@ define([ let val = $(this).val(); if (val !== '') { // enable xticks_label - $(that.wrapSelector('#xticks_label')).prop('readonly', false); + $(that.wrapSelector('#xticks_label')).attr('readonly', false); } else { // disable xticks_label - $(that.wrapSelector('#xticks_label')).prop('readonly', true); + $(that.wrapSelector('#xticks_label')).attr('readonly', true); } }); $(this.wrapSelector('#yticks')).on('change', function() { let val = $(this).val(); if (val !== '') { // enable yticks_label - $(that.wrapSelector('#yticks_label')).prop('readonly', false); + $(that.wrapSelector('#yticks_label')).attr('readonly', false); } else { // disable yticks_label - $(that.wrapSelector('#yticks_label')).prop('readonly', true); + $(that.wrapSelector('#yticks_label')).attr('readonly', true); } }); // axes - ticks label: inform user to type location option to use label - $(this.wrapSelector('#xticks_label[readonly]')).on('click', function() { - $(that.wrapSelector('#xticks')).focus(); + $(this.wrapSelector('#xticks_label')).on('click', function() { + if ($(that.wrapSelector('#xticks')).val() === '') { + $(that.wrapSelector('#xticks')).focus(); + } }); - $(this.wrapSelector('#yticks_label[readonly]')).on('click', function() { - $(that.wrapSelector('#yticks')).focus(); + $(this.wrapSelector('#yticks_label')).on('click', function() { + if ($(that.wrapSelector('#yticks')).val() === '') { + $(that.wrapSelector('#yticks')).focus(); + } }); // preview refresh @@ -519,6 +533,15 @@ define([ }); $(page).find('#sampleCount').html(sampleCountTag.toString()); + // set errorbar list + var vpErrorbarSuggest = new SuggestInput(); + vpErrorbarSuggest.setComponentID('errorbar'); + vpErrorbarSuggest.addClass('vp-input vp-state'); + vpErrorbarSuggest.setPlaceholder("('ci', 95)"); + vpErrorbarSuggest.setValue(this.state.errorbar); + vpErrorbarSuggest.setSuggestList(["None", "'ci'", "'pi'", "'sd'", "'se'"]); + $(page).find('#errorbar').replaceWith(vpErrorbarSuggest.toTagString()); + // data options depend on chart type $(page).find('.sb-option').hide(); if (this.state.chartType == 'histplot') { @@ -526,7 +549,9 @@ define([ $(page).find('#kde').closest('.sb-option').show(); $(page).find('#stat').closest('.sb-option').show(); } else if (this.state.chartType == 'barplot') { + $(page).find('#orient').closest('.sb-option').show(); $(page).find('#showValues').closest('.sb-option').show(); + $(page).find('#errorbar').closest('.sb-option').show(); if (this.state.setXY === false) { if (this.state.x !== '' && this.state.y !== '') { $(page).find('#sortBy').closest('.sb-option').show(); @@ -545,6 +570,10 @@ define([ $(page).find('#sortHue').closest('.sb-option').show(); } } + } else if (this.state.chartType == 'heatmap') { + $(page).find('#annot').closest('.sb-option').show(); + } else if (this.state.chartType === 'lineplot') { + $(page).find('#errorbar').closest('.sb-option').show(); } //================================================================ @@ -761,14 +790,16 @@ define([ handleInnerOk() { // generateImportCode var code = this.generateImportCode(); + // DEPRECATED: no longer save to block as default // create block and run it - $('#vp_wrapper').trigger({ - type: 'create_option_page', - blockType: 'block', - menuId: 'lgExe_code', - menuState: { taskState: { code: code } }, - afterAction: 'run' - }); + // $('#vp_wrapper').trigger({ + // type: 'create_option_page', + // blockType: 'block', + // menuId: 'lgExe_code', + // menuState: { taskState: { code: code } }, + // afterAction: 'run' + // }); + com_interface.insertCell('code', code, true, 'Visualization > Seaborn'); this.closeInnerPopup(); @@ -805,20 +836,20 @@ define([ } } else { var errorContent = ''; - if (msg.content.ename) { + if (msg.content && msg.content.ename) { errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue, msg.content.detail); } $(that.wrapSelector('#chartPreview')).html(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); } }).catch(function(resultObj) { let { msg } = resultObj; var errorContent = ''; - if (msg.content.ename) { + if (msg.content && msg.content.ename) { errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue, msg.content.detail); } $(that.wrapSelector('#chartPreview')).html(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); }); }); } @@ -862,8 +893,8 @@ define([ generateCode(preview=false) { let { - chartType, data, x, y, setXY, hue, kde, stat, - showValues, showValuesPrecision, + chartType, data, x, y, setXY, hue, orient, kde, stat, + showValues, showValuesPrecision, errorbar, sortType, sortBy, sortHue, sortHueText, userOption='', x_limit_from, x_limit_to, y_limit_from, y_limit_to, @@ -926,7 +957,14 @@ define([ etcOptionCode.push(com_util.formatString("marker='{0}'", markerStyle)); } if (showValues === true && chartType === 'barplot') { - etcOptionCode.push('ci=None'); + // etcOptionCode.push('ci=None'); // changed to errorbar after 0.12 version + etcOptionCode.push('errorbar=None'); + } else { + if (chartType === 'barplot' || chartType === 'lineplot') { + if (errorbar !== '') { + etcOptionCode.push(com_util.formatString("errorbar={0}", errorbar)); + } + } } if (setXY === false && sortType !== '') { let sortCode = ''; @@ -939,10 +977,10 @@ define([ } if (chartType === 'barplot' && sortX !== '' && sortY !== '') { if (hue !== '' && sortHue !== '') { - sortCode = com_util.formatString("{0}[{1}[{2}]=={3}].groupby({4})[{5}].mean().sort_values({6}).index" + sortCode = com_util.formatString("{0}[{1}[{2}]=={3}].groupby({4})[{5}].mean(numeric_only=True).sort_values({6}).index" , state.data, state.data, state.hue, com_util.convertToStr(sortHue, sortHueText), sortX, sortY, sortTypeStr); } else { - sortCode = com_util.formatString("{0}.groupby({1})[{2}].mean().sort_values({3}).index", state.data, sortX, sortY, sortTypeStr); + sortCode = com_util.formatString("{0}.groupby({1})[{2}].mean(numeric_only=True).sort_values({3}).index", state.data, sortX, sortY, sortTypeStr); } } else if (chartType === 'countplot' && (sortX !== '' || sortY !== '')) { let countVar = sortX === ''? sortY: sortX; diff --git a/visualpython/js/m_visualize/WordCloud.js b/visualpython/js/m_visualize/WordCloud.js index b361ed65..96ee22a1 100644 --- a/visualpython/js/m_visualize/WordCloud.js +++ b/visualpython/js/m_visualize/WordCloud.js @@ -32,6 +32,7 @@ define([ this.config.importButton = true; this.config.dataview = false; this.config.checkModules = ['Counter', 'plt', 'WordCloud']; + this.config.docs = 'https://amueller.github.io/word_cloud/references.html'; this.state = { data: '', @@ -264,20 +265,20 @@ define([ } } else { var errorContent = ''; - if (msg.content.ename) { + if (msg.content && msg.content.ename) { errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue); } $(that.wrapSelector('#vp_wcPreview')).html(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); } }).catch(function(resultObj) { let { msg } = resultObj; var errorContent = ''; - if (msg.content.ename) { + if (msg.content && msg.content.ename) { errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue); } $(that.wrapSelector('#vp_wcPreview')).html(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); }); }); } diff --git a/visualpython/js/menu/MenuFrame.js b/visualpython/js/menu/MenuFrame.js index 76aab784..386916a3 100644 --- a/visualpython/js/menu/MenuFrame.js +++ b/visualpython/js/menu/MenuFrame.js @@ -23,13 +23,14 @@ define([ '../com/component/Component', '../com/component/SuggestInput', '../com/component/InnerFuncViewer', + '../com/component/PackageManager', __VP_RAW_LOADER__('../../data/libraries.json'), // INTEGRATION: text! to raw-loader './MenuGroup', './MenuItem', './TaskBar' -], function(menuFrameHtml, menuFrameCss, com_Config, com_Const, com_util, com_interface, Component, SuggestInput, InnerFuncViewer, +], function(menuFrameHtml, menuFrameCss, com_Config, com_Const, com_util, com_interface, Component, SuggestInput, InnerFuncViewer, PackageManager, librariesJson, MenuGroup, MenuItem, TaskBar) { 'use strict'; @@ -114,6 +115,11 @@ define([ $(this.wrapSelector('#vp_versionUpdater')).on('click', function() { vpConfig.checkVpVersion(); }); + // Click package manager + $(this.wrapSelector('#vp_packageManager')).on('click', function() { + let packageManager = new PackageManager(); + packageManager.open(); + }); } _unbindResizable() { @@ -182,7 +188,7 @@ define([ getMenuLibraries() { var libraries = {}; // LAB: webpack5 load json object by default - if (vpConfig.extensionType === 'lab') { + if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { libraries = librariesJson; } else { libraries = JSON.parse(librariesJson); @@ -199,19 +205,19 @@ define([ return {}; } - getMenuLibrary(menuId, libraries=this.menuLibraries) { + getMenuLibrary(menuId, libraries=this.menuLibrariesFlatten) { for (var i=0; i < libraries.length; i++) { var item = libraries[i]; if (item) { if (item.id === menuId) { return item; } - if (item.type === 'package') { - var result = this.getMenuLibrary(menuId, item.item); - if (result) { - return result; - } - } + // if (item.type === 'package') { + // var result = this.getMenuLibrary(menuId, item.item); + // if (result) { + // return result; + // } + // } } } return null; @@ -236,6 +242,9 @@ define([ // } // } // this.$pageDom = $(replaceHtml); + + // add hover title for logo + this.$pageDom.find('.vp-logo').prop('title', com_Const.TOOLBAR_BTN_INFO.HELP); return this.$pageDom; } @@ -243,7 +252,12 @@ define([ var that = this; var body = group.getBody(); var item = group.getItem(); + var state = group.getState(); item && item.forEach(child => { + // remember parent to navigate its parent menu + var category = state.category?(state.category + ' > ' + state.name):state.name; + child['category'] = category; + if (child.type == 'package') { // packages : MenuGroup var menuGroup = new MenuGroup($(body), child); @@ -252,8 +266,8 @@ define([ } } else { // functions : MenuItem + that.menuLibrariesFlatten.push(child); if (!child.hide) { - that.menuLibrariesFlatten.push(child); var menuItem = new MenuItem($(body), child); } } @@ -289,7 +303,7 @@ define([ }); let functionList = this.menuLibrariesFlatten.map(menu => { - return { label: menu.name, value: menu.name, ...menu } + return { label: menu.name, value: menu.name, dtype: menu.category, ...menu } }); // render searchbox let searchBox = new SuggestInput(); diff --git a/visualpython/js/menu/MenuItem.js b/visualpython/js/menu/MenuItem.js index 22343b2e..8cebdd73 100644 --- a/visualpython/js/menu/MenuItem.js +++ b/visualpython/js/menu/MenuItem.js @@ -148,6 +148,10 @@ define([ // LAB: img to url // page.appendFormatLine('

', com_Const.IMAGE_PATH + apps.icon); page.appendFormatLine('

', id); + // Exception for title alignment + if (id === 'stats_studentstTest') { + name = "Student's
t-test"; + } page.appendFormatLine('

{0}

', name); page.append('